@moxn/kb-migrate 0.4.40 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blocks-to-grammar.d.ts +51 -0
- package/dist/blocks-to-grammar.js +133 -0
- package/dist/client.d.ts +15 -4
- package/dist/client.js +70 -117
- package/dist/migrate-document.test.js +100 -20
- package/dist/sources/local.js +8 -3
- package/dist/sources/notion-blocks.js +1 -1
- package/dist/sources/notion.js +1 -1
- package/dist/sources/onenote/__tests__/onenote-tree.test.d.ts +1 -0
- package/dist/sources/onenote/__tests__/onenote-tree.test.js +43 -0
- package/dist/sources/onenote/onenote-api.d.ts +0 -1
- package/package.json +1 -1
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ContentBlock[] → grammar markdown serializer (Phase B of the write-surface
|
|
3
|
+
* tightening: the REST block-input path is retired; every import lands via
|
|
4
|
+
* `import_markdown`, the same grammar the export emits and the app's codec
|
|
5
|
+
* round-trips).
|
|
6
|
+
*
|
|
7
|
+
* The extractors (Notion / OneNote) keep producing ContentBlock sections —
|
|
8
|
+
* that stays a useful internal representation. This module is the WRITE
|
|
9
|
+
* BOUNDARY: it assumes media blocks have already been uploaded/re-hosted to
|
|
10
|
+
* storage keys (see `MoxnClient.processSections`, which now also re-hosts
|
|
11
|
+
* `url`/`base64` media) and reduces the sections to the `## H2` + `:::`
|
|
12
|
+
* directive grammar.
|
|
13
|
+
*
|
|
14
|
+
* Directive attr serialization REPLICATES the app codec
|
|
15
|
+
* (`src/lib/kb/grammar/attrs.ts` — kb-migrate is a separate package and
|
|
16
|
+
* cannot import it, same posture as grammar-media.ts): string → double-quoted
|
|
17
|
+
* with `&`→`&`, `"`→`"`; number → bare; string[] → `[a,b]` with
|
|
18
|
+
* `\` escaping for `\`, `,`, `]`.
|
|
19
|
+
*/
|
|
20
|
+
import type { ContentBlock } from './types.js';
|
|
21
|
+
type AttrValue = string | number | string[] | null | undefined;
|
|
22
|
+
/** Mirror of the app codec's `serializeAttrs` (src/lib/kb/grammar/attrs.ts). */
|
|
23
|
+
export declare function serializeAttrs(obj: Record<string, AttrValue>): string;
|
|
24
|
+
/**
|
|
25
|
+
* One ContentBlock → grammar chunk (markdown text or a `:::` directive line).
|
|
26
|
+
* Returns null for a block that cannot be expressed (e.g. base64 media that
|
|
27
|
+
* was not uploaded — the caller is expected to have re-hosted those first).
|
|
28
|
+
*/
|
|
29
|
+
export declare function blockToGrammar(block: ContentBlock): string | null;
|
|
30
|
+
export interface SectionsToGrammarResult {
|
|
31
|
+
markdown: string;
|
|
32
|
+
/** Media blocks that could not be expressed (no storage key / URL). */
|
|
33
|
+
dropped: number;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Sections → grammar markdown BODY (no front-matter — the caller prepends
|
|
37
|
+
* it). Each section becomes an `## <name>` chunk; blocks join with blank
|
|
38
|
+
* lines, matching the H2-split the server applies on the way back in.
|
|
39
|
+
*/
|
|
40
|
+
export declare function sectionsToGrammarMarkdown(sections: Array<{
|
|
41
|
+
name: string;
|
|
42
|
+
content: ContentBlock[];
|
|
43
|
+
}>): SectionsToGrammarResult;
|
|
44
|
+
/**
|
|
45
|
+
* Minimal YAML front-matter for an import: `name` (+ optional description).
|
|
46
|
+
* `import_markdown` HONORS the front-matter (replace_document semantics on
|
|
47
|
+
* update — the import source is the source of truth for metadata).
|
|
48
|
+
* Values are JSON-quoted, which is valid YAML for scalar strings.
|
|
49
|
+
*/
|
|
50
|
+
export declare function buildImportFrontMatter(name: string, description?: string | null): string;
|
|
51
|
+
export {};
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ContentBlock[] → grammar markdown serializer (Phase B of the write-surface
|
|
3
|
+
* tightening: the REST block-input path is retired; every import lands via
|
|
4
|
+
* `import_markdown`, the same grammar the export emits and the app's codec
|
|
5
|
+
* round-trips).
|
|
6
|
+
*
|
|
7
|
+
* The extractors (Notion / OneNote) keep producing ContentBlock sections —
|
|
8
|
+
* that stays a useful internal representation. This module is the WRITE
|
|
9
|
+
* BOUNDARY: it assumes media blocks have already been uploaded/re-hosted to
|
|
10
|
+
* storage keys (see `MoxnClient.processSections`, which now also re-hosts
|
|
11
|
+
* `url`/`base64` media) and reduces the sections to the `## H2` + `:::`
|
|
12
|
+
* directive grammar.
|
|
13
|
+
*
|
|
14
|
+
* Directive attr serialization REPLICATES the app codec
|
|
15
|
+
* (`src/lib/kb/grammar/attrs.ts` — kb-migrate is a separate package and
|
|
16
|
+
* cannot import it, same posture as grammar-media.ts): string → double-quoted
|
|
17
|
+
* with `&`→`&`, `"`→`"`; number → bare; string[] → `[a,b]` with
|
|
18
|
+
* `\` escaping for `\`, `,`, `]`.
|
|
19
|
+
*/
|
|
20
|
+
function escapeArrayElement(v) {
|
|
21
|
+
return v.replace(/\\/g, '\\\\').replace(/,/g, '\\,').replace(/]/g, '\\]');
|
|
22
|
+
}
|
|
23
|
+
/** Mirror of the app codec's `serializeAttrs` (src/lib/kb/grammar/attrs.ts). */
|
|
24
|
+
export function serializeAttrs(obj) {
|
|
25
|
+
const parts = [];
|
|
26
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
27
|
+
if (value === null || value === undefined)
|
|
28
|
+
continue;
|
|
29
|
+
if (Array.isArray(value)) {
|
|
30
|
+
parts.push(`${key}=[${value.map(escapeArrayElement).join(',')}]`);
|
|
31
|
+
}
|
|
32
|
+
else if (typeof value === 'number') {
|
|
33
|
+
parts.push(`${key}=${value}`);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
const escaped = value.replace(/&/g, '&').replace(/"/g, '"');
|
|
37
|
+
parts.push(`${key}="${escaped}"`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return parts.join(' ');
|
|
41
|
+
}
|
|
42
|
+
/** The `ref` a media block serializes: storage key first, else its URL. */
|
|
43
|
+
function blockRef(block) {
|
|
44
|
+
if (block.type === 'storage' && block.key)
|
|
45
|
+
return block.key;
|
|
46
|
+
if (block.type === 'url' && block.url)
|
|
47
|
+
return block.url;
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* One ContentBlock → grammar chunk (markdown text or a `:::` directive line).
|
|
52
|
+
* Returns null for a block that cannot be expressed (e.g. base64 media that
|
|
53
|
+
* was not uploaded — the caller is expected to have re-hosted those first).
|
|
54
|
+
*/
|
|
55
|
+
export function blockToGrammar(block) {
|
|
56
|
+
switch (block.blockType) {
|
|
57
|
+
case 'text':
|
|
58
|
+
return block.text;
|
|
59
|
+
case 'image': {
|
|
60
|
+
const ref = blockRef(block);
|
|
61
|
+
if (!ref)
|
|
62
|
+
return null;
|
|
63
|
+
return `:::image{${serializeAttrs({
|
|
64
|
+
ref,
|
|
65
|
+
alt: block.alt,
|
|
66
|
+
mime: block.mediaType,
|
|
67
|
+
})}}`;
|
|
68
|
+
}
|
|
69
|
+
case 'document':
|
|
70
|
+
case 'file': {
|
|
71
|
+
const ref = blockRef(block);
|
|
72
|
+
if (!ref)
|
|
73
|
+
return null;
|
|
74
|
+
return `:::file{${serializeAttrs({
|
|
75
|
+
ref,
|
|
76
|
+
filename: 'filename' in block ? block.filename : undefined,
|
|
77
|
+
mime: block.mediaType,
|
|
78
|
+
})}}`;
|
|
79
|
+
}
|
|
80
|
+
case 'csv': {
|
|
81
|
+
const ref = blockRef(block);
|
|
82
|
+
if (!ref)
|
|
83
|
+
return null;
|
|
84
|
+
return `:::csv{${serializeAttrs({
|
|
85
|
+
ref,
|
|
86
|
+
filename: block.filename,
|
|
87
|
+
mime: 'text/csv',
|
|
88
|
+
headers: block.headers,
|
|
89
|
+
rowCount: block.rowCount,
|
|
90
|
+
})}}`;
|
|
91
|
+
}
|
|
92
|
+
case 'database_embed':
|
|
93
|
+
return `:::db{${serializeAttrs({ ref: block.databaseId })}}`;
|
|
94
|
+
default:
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Sections → grammar markdown BODY (no front-matter — the caller prepends
|
|
100
|
+
* it). Each section becomes an `## <name>` chunk; blocks join with blank
|
|
101
|
+
* lines, matching the H2-split the server applies on the way back in.
|
|
102
|
+
*/
|
|
103
|
+
export function sectionsToGrammarMarkdown(sections) {
|
|
104
|
+
let dropped = 0;
|
|
105
|
+
const chunks = [];
|
|
106
|
+
for (const section of sections) {
|
|
107
|
+
const parts = [`## ${section.name}`];
|
|
108
|
+
for (const block of section.content) {
|
|
109
|
+
const chunk = blockToGrammar(block);
|
|
110
|
+
if (chunk === null) {
|
|
111
|
+
dropped++;
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (chunk.trim().length > 0)
|
|
115
|
+
parts.push(chunk.trim());
|
|
116
|
+
}
|
|
117
|
+
chunks.push(parts.join('\n\n'));
|
|
118
|
+
}
|
|
119
|
+
return { markdown: chunks.join('\n\n'), dropped };
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Minimal YAML front-matter for an import: `name` (+ optional description).
|
|
123
|
+
* `import_markdown` HONORS the front-matter (replace_document semantics on
|
|
124
|
+
* update — the import source is the source of truth for metadata).
|
|
125
|
+
* Values are JSON-quoted, which is valid YAML for scalar strings.
|
|
126
|
+
*/
|
|
127
|
+
export function buildImportFrontMatter(name, description) {
|
|
128
|
+
const lines = [`name: ${JSON.stringify(name)}`];
|
|
129
|
+
if (description !== undefined && description !== null && description !== '') {
|
|
130
|
+
lines.push(`description: ${JSON.stringify(description)}`);
|
|
131
|
+
}
|
|
132
|
+
return `---\n${lines.join('\n')}\n---\n\n`;
|
|
133
|
+
}
|
package/dist/client.d.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* API client for Moxn KB
|
|
3
3
|
*/
|
|
4
|
-
/// <reference types="node" resolution-mode="require"/>
|
|
5
4
|
import type { ExtractedDocument, MigrationOptions, MigrationResult, DocumentListItem, DocumentDetail, ExportOptions } from './types.js';
|
|
6
5
|
import type { DateFilter } from './date-filter.js';
|
|
7
6
|
/**
|
|
@@ -33,6 +32,10 @@ export interface ImportMarkdownResult {
|
|
|
33
32
|
* lenient fallback recovered the name. Carries the js-yaml error message.
|
|
34
33
|
*/
|
|
35
34
|
frontMatterError?: string;
|
|
35
|
+
/** Section anchor ids in POSITION ORDER after the upsert (for cross-refs). */
|
|
36
|
+
sectionIds?: string[];
|
|
37
|
+
/** The branch the upsert landed on (for branch-scoped follow-ups like tags). */
|
|
38
|
+
branchId?: string;
|
|
36
39
|
}
|
|
37
40
|
export declare class MoxnClient {
|
|
38
41
|
private apiUrl;
|
|
@@ -43,6 +46,17 @@ export declare class MoxnClient {
|
|
|
43
46
|
/**
|
|
44
47
|
* Migrate a single document
|
|
45
48
|
*/
|
|
49
|
+
/**
|
|
50
|
+
* Migrate one extracted document via `import_markdown` (Phase B of the
|
|
51
|
+
* write-surface tightening — the REST block-input path is retired).
|
|
52
|
+
*
|
|
53
|
+
* Flow: upload/re-host every media block (local files, base64 payloads,
|
|
54
|
+
* remote URLs) → serialize the sections to grammar markdown → prepend a
|
|
55
|
+
* name/description front-matter → UPSERT via `import_markdown`
|
|
56
|
+
* (create-or-replace_document; `onConflict: 'skip'` skips an existing
|
|
57
|
+
* path). Idempotent: a byte-identical re-import is reported as a no-op
|
|
58
|
+
* update by the server.
|
|
59
|
+
*/
|
|
46
60
|
migrateDocument(doc: ExtractedDocument, basePath: string, onConflict: 'skip' | 'update', dryRun: boolean): Promise<MigrationResult>;
|
|
47
61
|
/**
|
|
48
62
|
* List all documents, optionally filtered by path prefix.
|
|
@@ -103,8 +117,6 @@ export declare class MoxnClient {
|
|
|
103
117
|
key: string;
|
|
104
118
|
}>;
|
|
105
119
|
private getUploadUrl;
|
|
106
|
-
private createDocument;
|
|
107
|
-
private updateDocument;
|
|
108
120
|
/**
|
|
109
121
|
* Create a KB database.
|
|
110
122
|
*/
|
|
@@ -264,5 +276,4 @@ export declare class MoxnClient {
|
|
|
264
276
|
id: string;
|
|
265
277
|
name: string;
|
|
266
278
|
} | null>;
|
|
267
|
-
private isConflictError;
|
|
268
279
|
}
|
package/dist/client.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import * as fs from 'fs/promises';
|
|
5
5
|
import { formatApiError } from './api-error.js';
|
|
6
|
+
import { sectionsToGrammarMarkdown, buildImportFrontMatter, } from './blocks-to-grammar.js';
|
|
6
7
|
export class MoxnClient {
|
|
7
8
|
apiUrl;
|
|
8
9
|
apiKey;
|
|
@@ -21,6 +22,17 @@ export class MoxnClient {
|
|
|
21
22
|
/**
|
|
22
23
|
* Migrate a single document
|
|
23
24
|
*/
|
|
25
|
+
/**
|
|
26
|
+
* Migrate one extracted document via `import_markdown` (Phase B of the
|
|
27
|
+
* write-surface tightening — the REST block-input path is retired).
|
|
28
|
+
*
|
|
29
|
+
* Flow: upload/re-host every media block (local files, base64 payloads,
|
|
30
|
+
* remote URLs) → serialize the sections to grammar markdown → prepend a
|
|
31
|
+
* name/description front-matter → UPSERT via `import_markdown`
|
|
32
|
+
* (create-or-replace_document; `onConflict: 'skip'` skips an existing
|
|
33
|
+
* path). Idempotent: a byte-identical re-import is reported as a no-op
|
|
34
|
+
* update by the server.
|
|
35
|
+
*/
|
|
24
36
|
async migrateDocument(doc, basePath, onConflict, dryRun) {
|
|
25
37
|
const startTime = Date.now();
|
|
26
38
|
const documentPath = this.buildPath(basePath, doc.relativePath);
|
|
@@ -34,91 +46,36 @@ export class MoxnClient {
|
|
|
34
46
|
};
|
|
35
47
|
}
|
|
36
48
|
try {
|
|
37
|
-
//
|
|
49
|
+
// Upload/re-host media (local paths, base64, remote URLs) → storage keys.
|
|
38
50
|
const processedSections = await this.processSections(doc.sections);
|
|
39
|
-
|
|
40
|
-
|
|
51
|
+
const { markdown: body, dropped } = sectionsToGrammarMarkdown(processedSections);
|
|
52
|
+
if (dropped > 0) {
|
|
53
|
+
console.error(` ! ${dropped} media block(s) had no uploadable payload and were dropped: ${doc.sourcePath}`);
|
|
54
|
+
}
|
|
55
|
+
const markdown = buildImportFrontMatter(doc.name, doc.description) + body;
|
|
56
|
+
const result = await this.importMarkdown({
|
|
57
|
+
markdown,
|
|
41
58
|
path: documentPath,
|
|
42
|
-
|
|
43
|
-
description: doc.description,
|
|
44
|
-
defaultPermission: this.defaultPermission,
|
|
45
|
-
aiAccess: this.aiAccess,
|
|
46
|
-
sections: processedSections,
|
|
59
|
+
onConflict,
|
|
47
60
|
});
|
|
48
61
|
return {
|
|
49
62
|
sourcePath: doc.sourcePath,
|
|
50
|
-
documentPath,
|
|
51
|
-
status: 'created'
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
63
|
+
documentPath: result.path,
|
|
64
|
+
status: result.outcome === 'created'
|
|
65
|
+
? 'created'
|
|
66
|
+
: result.outcome === 'updated'
|
|
67
|
+
? 'updated'
|
|
68
|
+
: 'skipped',
|
|
69
|
+
documentId: result.id,
|
|
70
|
+
branchId: result.branchId,
|
|
71
|
+
sectionsCount: result.sectionIds?.length,
|
|
72
|
+
sectionIds: result.sectionIds,
|
|
56
73
|
references: doc.references,
|
|
57
74
|
sourcePageId: doc.metadata?.notionPageId,
|
|
58
75
|
duration: Date.now() - startTime,
|
|
59
76
|
};
|
|
60
77
|
}
|
|
61
78
|
catch (error) {
|
|
62
|
-
// Check for conflict (409)
|
|
63
|
-
if (this.isConflictError(error)) {
|
|
64
|
-
if (onConflict === 'skip') {
|
|
65
|
-
return {
|
|
66
|
-
sourcePath: doc.sourcePath,
|
|
67
|
-
documentPath,
|
|
68
|
-
status: 'skipped',
|
|
69
|
-
documentId: error.documentId,
|
|
70
|
-
branchId: error.branchId,
|
|
71
|
-
duration: Date.now() - startTime,
|
|
72
|
-
};
|
|
73
|
-
}
|
|
74
|
-
// Update existing document
|
|
75
|
-
try {
|
|
76
|
-
const processedSections = await this.processSections(doc.sections);
|
|
77
|
-
const updateResult = await this.updateDocument(error.documentId, {
|
|
78
|
-
name: doc.name,
|
|
79
|
-
description: doc.description,
|
|
80
|
-
sections: processedSections,
|
|
81
|
-
});
|
|
82
|
-
return {
|
|
83
|
-
sourcePath: doc.sourcePath,
|
|
84
|
-
documentPath,
|
|
85
|
-
status: 'updated',
|
|
86
|
-
documentId: updateResult.id,
|
|
87
|
-
branchId: updateResult.branchId,
|
|
88
|
-
sectionsCount: updateResult.sections.length,
|
|
89
|
-
sectionIds: updateResult.sections.map((s) => s.id),
|
|
90
|
-
references: doc.references,
|
|
91
|
-
sourcePageId: doc.metadata?.notionPageId,
|
|
92
|
-
duration: Date.now() - startTime,
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
catch (updateError) {
|
|
96
|
-
// Re-importing BYTE-IDENTICAL content makes the server reject the
|
|
97
|
-
// empty commit ("No changes to commit"). For an importer that's a
|
|
98
|
-
// NO-OP, not a failure — report a successful (no-op) update, matching
|
|
99
|
-
// the grammar path's idempotent re-import. (Audit finding B.)
|
|
100
|
-
const msg = updateError instanceof Error ? updateError.message : '';
|
|
101
|
-
if (msg.includes('No changes to commit')) {
|
|
102
|
-
return {
|
|
103
|
-
sourcePath: doc.sourcePath,
|
|
104
|
-
documentPath,
|
|
105
|
-
status: 'updated',
|
|
106
|
-
documentId: error.documentId,
|
|
107
|
-
branchId: error.branchId,
|
|
108
|
-
sourcePageId: doc.metadata?.notionPageId,
|
|
109
|
-
duration: Date.now() - startTime,
|
|
110
|
-
};
|
|
111
|
-
}
|
|
112
|
-
return {
|
|
113
|
-
sourcePath: doc.sourcePath,
|
|
114
|
-
documentPath,
|
|
115
|
-
status: 'failed',
|
|
116
|
-
documentId: error.documentId,
|
|
117
|
-
error: updateError instanceof Error ? updateError.message : 'Update failed',
|
|
118
|
-
duration: Date.now() - startTime,
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
79
|
return {
|
|
123
80
|
sourcePath: doc.sourcePath,
|
|
124
81
|
documentPath,
|
|
@@ -358,6 +315,45 @@ export class MoxnClient {
|
|
|
358
315
|
filename: block.filename,
|
|
359
316
|
};
|
|
360
317
|
}
|
|
318
|
+
// Base64 payloads (extractors that inline-downloaded media) → upload.
|
|
319
|
+
if ((block.blockType === 'image' ||
|
|
320
|
+
block.blockType === 'document' ||
|
|
321
|
+
block.blockType === 'csv' ||
|
|
322
|
+
block.blockType === 'file') &&
|
|
323
|
+
block.type === 'base64' &&
|
|
324
|
+
block.base64) {
|
|
325
|
+
const data = Buffer.from(block.base64, 'base64');
|
|
326
|
+
const filename = ('filename' in block ? block.filename : undefined) || 'media';
|
|
327
|
+
const { key } = await this.uploadFile(data, block.mediaType || 'application/octet-stream', filename);
|
|
328
|
+
return { ...block, type: 'storage', key, base64: undefined };
|
|
329
|
+
}
|
|
330
|
+
// Remote URLs (e.g. expiring Notion signed URLs) → download + re-host,
|
|
331
|
+
// matching the durability the server-side blocksToTipTap re-hosting
|
|
332
|
+
// used to provide. A failed download keeps the URL ref (the grammar
|
|
333
|
+
// carries external URLs) rather than failing the document.
|
|
334
|
+
if ((block.blockType === 'image' ||
|
|
335
|
+
block.blockType === 'document' ||
|
|
336
|
+
block.blockType === 'csv' ||
|
|
337
|
+
block.blockType === 'file') &&
|
|
338
|
+
block.type === 'url' &&
|
|
339
|
+
block.url) {
|
|
340
|
+
try {
|
|
341
|
+
const response = await fetch(block.url);
|
|
342
|
+
if (!response.ok)
|
|
343
|
+
throw new Error(`HTTP ${response.status}`);
|
|
344
|
+
const data = Buffer.from(await response.arrayBuffer());
|
|
345
|
+
const filename = ('filename' in block ? block.filename : undefined) ||
|
|
346
|
+
new URL(block.url).pathname.split('/').pop() ||
|
|
347
|
+
'media';
|
|
348
|
+
const { key } = await this.uploadFile(data, block.mediaType || 'application/octet-stream', filename);
|
|
349
|
+
return { ...block, type: 'storage', key, url: undefined };
|
|
350
|
+
}
|
|
351
|
+
catch (err) {
|
|
352
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
353
|
+
console.error(` ! media re-host failed, keeping external URL ref: ${block.url}: ${msg}`);
|
|
354
|
+
return block;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
361
357
|
return block;
|
|
362
358
|
}));
|
|
363
359
|
}
|
|
@@ -394,42 +390,6 @@ export class MoxnClient {
|
|
|
394
390
|
const data = await response.json();
|
|
395
391
|
return { key: data.key, uploadUrl: data.uploadUrl };
|
|
396
392
|
}
|
|
397
|
-
async createDocument(request) {
|
|
398
|
-
const response = await fetch(`${this.apiUrl}/api/v1/kb/documents`, {
|
|
399
|
-
method: 'POST',
|
|
400
|
-
headers: {
|
|
401
|
-
'Content-Type': 'application/json',
|
|
402
|
-
'x-api-key': this.apiKey,
|
|
403
|
-
},
|
|
404
|
-
body: JSON.stringify(request),
|
|
405
|
-
});
|
|
406
|
-
if (!response.ok) {
|
|
407
|
-
const body = await response.json().catch(() => ({}));
|
|
408
|
-
if (response.status === 409 && body.documentId) {
|
|
409
|
-
const error = new Error(body.error || 'Document already exists');
|
|
410
|
-
error.documentId = body.documentId;
|
|
411
|
-
error.branchId = body.branchId;
|
|
412
|
-
throw error;
|
|
413
|
-
}
|
|
414
|
-
throw new Error(formatApiError(response.status, body));
|
|
415
|
-
}
|
|
416
|
-
return response.json();
|
|
417
|
-
}
|
|
418
|
-
async updateDocument(documentId, request) {
|
|
419
|
-
const response = await fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}`, {
|
|
420
|
-
method: 'PUT',
|
|
421
|
-
headers: {
|
|
422
|
-
'Content-Type': 'application/json',
|
|
423
|
-
'x-api-key': this.apiKey,
|
|
424
|
-
},
|
|
425
|
-
body: JSON.stringify(request),
|
|
426
|
-
});
|
|
427
|
-
if (!response.ok) {
|
|
428
|
-
const body = await response.json().catch(() => ({}));
|
|
429
|
-
throw new Error(formatApiError(response.status, body));
|
|
430
|
-
}
|
|
431
|
-
return response.json();
|
|
432
|
-
}
|
|
433
393
|
// ──────────────────────────────────────────────
|
|
434
394
|
// Database & tag methods (for Notion import)
|
|
435
395
|
// ──────────────────────────────────────────────
|
|
@@ -670,11 +630,4 @@ export class MoxnClient {
|
|
|
670
630
|
const databases = await this.listDatabases();
|
|
671
631
|
return databases.find((db) => db.id === databaseId) ?? null;
|
|
672
632
|
}
|
|
673
|
-
isConflictError(error) {
|
|
674
|
-
return (error instanceof Error &&
|
|
675
|
-
'documentId' in error &&
|
|
676
|
-
'branchId' in error &&
|
|
677
|
-
typeof error.documentId === 'string' &&
|
|
678
|
-
typeof error.branchId === 'string');
|
|
679
|
-
}
|
|
680
633
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { describe, it, expect, vi, afterEach } from 'vitest';
|
|
2
2
|
import { MoxnClient } from './client.js';
|
|
3
|
+
import { sectionsToGrammarMarkdown, buildImportFrontMatter, blockToGrammar, } from './blocks-to-grammar.js';
|
|
3
4
|
/** Minimal fetch Response stand-in (client reads .ok/.status/.json()). */
|
|
4
5
|
function mockResponse(status, body) {
|
|
5
6
|
return {
|
|
@@ -25,22 +26,59 @@ function client() {
|
|
|
25
26
|
});
|
|
26
27
|
}
|
|
27
28
|
afterEach(() => vi.restoreAllMocks());
|
|
28
|
-
describe('migrateDocument —
|
|
29
|
-
it(
|
|
30
|
-
|
|
29
|
+
describe('migrateDocument — grammar-markdown import (write-surface Phase B)', () => {
|
|
30
|
+
it('serializes sections to grammar markdown and UPSERTs via import_markdown', async () => {
|
|
31
|
+
let importBody;
|
|
31
32
|
vi.spyOn(globalThis, 'fetch').mockImplementation((async (url, init) => {
|
|
32
33
|
const u = String(url);
|
|
33
34
|
const method = (init?.method ?? 'GET').toUpperCase();
|
|
34
|
-
if (method === 'POST' && u.endsWith('/api/v1/kb/
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
35
|
+
if (method === 'POST' && u.endsWith('/api/v1/kb/import')) {
|
|
36
|
+
importBody = JSON.parse(String(init?.body));
|
|
37
|
+
return mockResponse(200, {
|
|
38
|
+
result: {
|
|
39
|
+
id: 'doc-1',
|
|
40
|
+
path: '/runbook',
|
|
41
|
+
outcome: 'created',
|
|
42
|
+
existed: false,
|
|
43
|
+
name: 'Runbook',
|
|
44
|
+
frontMatterIgnored: false,
|
|
45
|
+
sectionIds: ['s1'],
|
|
46
|
+
branchId: 'br-1',
|
|
47
|
+
},
|
|
39
48
|
});
|
|
40
49
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
50
|
+
return mockResponse(200, {});
|
|
51
|
+
}));
|
|
52
|
+
const res = await client().migrateDocument(doc, '/', 'update', false);
|
|
53
|
+
expect(res.status).toBe('created');
|
|
54
|
+
expect(res.documentId).toBe('doc-1');
|
|
55
|
+
expect(res.branchId).toBe('br-1');
|
|
56
|
+
expect(res.sectionIds).toEqual(['s1']);
|
|
57
|
+
// POSITIVE wire assertion: the retired blocks payload is GONE; the body is
|
|
58
|
+
// grammar markdown with the name front-matter + H2 sections.
|
|
59
|
+
expect(importBody?.action).toBe('import_markdown');
|
|
60
|
+
const md = String(importBody?.markdown);
|
|
61
|
+
expect(md).toContain('name: "Runbook"');
|
|
62
|
+
expect(md).toContain('## Steps');
|
|
63
|
+
expect(md).toContain('body');
|
|
64
|
+
expect(importBody).not.toHaveProperty('sections');
|
|
65
|
+
});
|
|
66
|
+
it("maps the server's no-op re-import (outcome 'updated') to an updated result", async () => {
|
|
67
|
+
// The old client-side "No changes to commit" special-case moved server-side:
|
|
68
|
+
// import_markdown swallows the empty-commit rejection and reports 'updated'.
|
|
69
|
+
vi.spyOn(globalThis, 'fetch').mockImplementation((async (url, init) => {
|
|
70
|
+
const u = String(url);
|
|
71
|
+
const method = (init?.method ?? 'GET').toUpperCase();
|
|
72
|
+
if (method === 'POST' && u.endsWith('/api/v1/kb/import')) {
|
|
73
|
+
return mockResponse(200, {
|
|
74
|
+
result: {
|
|
75
|
+
id: 'doc-1',
|
|
76
|
+
path: '/runbook',
|
|
77
|
+
outcome: 'updated',
|
|
78
|
+
existed: true,
|
|
79
|
+
name: 'Runbook',
|
|
80
|
+
frontMatterIgnored: false,
|
|
81
|
+
},
|
|
44
82
|
});
|
|
45
83
|
}
|
|
46
84
|
return mockResponse(200, {});
|
|
@@ -49,22 +87,64 @@ describe('migrateDocument — finding B: unchanged re-import is a no-op, not a f
|
|
|
49
87
|
expect(res.status).toBe('updated'); // NOT 'failed'
|
|
50
88
|
expect(res.documentId).toBe('doc-1');
|
|
51
89
|
});
|
|
52
|
-
it('still reports a genuine
|
|
90
|
+
it('still reports a genuine import error as failed', async () => {
|
|
53
91
|
vi.spyOn(globalThis, 'fetch').mockImplementation((async (url, init) => {
|
|
54
92
|
const u = String(url);
|
|
55
93
|
const method = (init?.method ?? 'GET').toUpperCase();
|
|
56
|
-
if (method === 'POST' && u.endsWith('/api/v1/kb/
|
|
57
|
-
return mockResponse(409, {
|
|
58
|
-
error: 'exists',
|
|
59
|
-
documentId: 'doc-1',
|
|
60
|
-
branchId: 'br-1',
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
if (method === 'PUT')
|
|
94
|
+
if (method === 'POST' && u.endsWith('/api/v1/kb/import')) {
|
|
64
95
|
return mockResponse(500, { error: 'boom' });
|
|
96
|
+
}
|
|
65
97
|
return mockResponse(200, {});
|
|
66
98
|
}));
|
|
67
99
|
const res = await client().migrateDocument(doc, '/', 'update', false);
|
|
68
100
|
expect(res.status).toBe('failed');
|
|
69
101
|
});
|
|
70
102
|
});
|
|
103
|
+
describe('blocks-to-grammar serializer', () => {
|
|
104
|
+
it('serializes media blocks to ::: directives (storage key first, url fallback)', () => {
|
|
105
|
+
expect(blockToGrammar({
|
|
106
|
+
blockType: 'image',
|
|
107
|
+
type: 'storage',
|
|
108
|
+
key: 'tenant_1/img.png',
|
|
109
|
+
mediaType: 'image/png',
|
|
110
|
+
alt: 'diagram',
|
|
111
|
+
})).toBe(':::image{ref="tenant_1/img.png" alt="diagram" mime="image/png"}');
|
|
112
|
+
expect(blockToGrammar({
|
|
113
|
+
blockType: 'image',
|
|
114
|
+
type: 'url',
|
|
115
|
+
url: 'https://example.test/pic.png',
|
|
116
|
+
mediaType: 'image/png',
|
|
117
|
+
})).toBe(':::image{ref="https://example.test/pic.png" mime="image/png"}');
|
|
118
|
+
expect(blockToGrammar({ blockType: 'database_embed', databaseId: 'db-1' })).toBe(':::db{ref="db-1"}');
|
|
119
|
+
expect(blockToGrammar({
|
|
120
|
+
blockType: 'csv',
|
|
121
|
+
type: 'storage',
|
|
122
|
+
key: 'tenant_1/data.csv',
|
|
123
|
+
mediaType: 'text/csv',
|
|
124
|
+
filename: 'data.csv',
|
|
125
|
+
headers: ['a', 'b'],
|
|
126
|
+
rowCount: 3,
|
|
127
|
+
})).toBe(':::csv{ref="tenant_1/data.csv" filename="data.csv" mime="text/csv" headers=[a,b] rowCount=3}');
|
|
128
|
+
});
|
|
129
|
+
it('drops media with no expressible ref and counts it', () => {
|
|
130
|
+
const { markdown, dropped } = sectionsToGrammarMarkdown([
|
|
131
|
+
{
|
|
132
|
+
name: 'S',
|
|
133
|
+
content: [
|
|
134
|
+
{ blockType: 'text', text: 'kept' },
|
|
135
|
+
{
|
|
136
|
+
blockType: 'image',
|
|
137
|
+
type: 'base64',
|
|
138
|
+
base64: 'xxxx',
|
|
139
|
+
mediaType: 'image/png',
|
|
140
|
+
},
|
|
141
|
+
],
|
|
142
|
+
},
|
|
143
|
+
]);
|
|
144
|
+
expect(dropped).toBe(1);
|
|
145
|
+
expect(markdown).toBe('## S\n\nkept');
|
|
146
|
+
});
|
|
147
|
+
it('front-matter escapes names safely (JSON-quoted scalars)', () => {
|
|
148
|
+
expect(buildImportFrontMatter('He said "hi": ok')).toBe('---\nname: "He said \\"hi\\": ok"\n---\n\n');
|
|
149
|
+
});
|
|
150
|
+
});
|
package/dist/sources/local.js
CHANGED
|
@@ -7,9 +7,14 @@ import * as fs from 'fs/promises';
|
|
|
7
7
|
import * as fsSync from 'fs';
|
|
8
8
|
import * as path from 'path';
|
|
9
9
|
import { glob } from 'glob';
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
import
|
|
10
|
+
// NAMED import, matching this package's OWN dependency (unified@^11, pure ESM,
|
|
11
|
+
// named export only — no default). The published artifact resolves ^11; a
|
|
12
|
+
// default import bricks the npx-installed CLI at load time (proven: 0.4.40).
|
|
13
|
+
// In-repo gotcha: this package is STANDALONE-installed (own package-lock.json)
|
|
14
|
+
// — run `npm install` INSIDE packages/kb-migrate before building, or tsc/tsx
|
|
15
|
+
// resolve the monorepo ROOT's hoisted CJS unified@9 and this import appears
|
|
16
|
+
// broken (TS2595). That mirage is what shipped the 0.4.40 regression.
|
|
17
|
+
import { unified } from 'unified';
|
|
13
18
|
import remarkParse from 'remark-parse';
|
|
14
19
|
import { MigrationSource } from './base.js';
|
|
15
20
|
/**
|
package/dist/sources/notion.js
CHANGED
|
@@ -13,7 +13,7 @@ import { blocksToSections, getPageTitle, normalizeId, } from './notion-blocks.js
|
|
|
13
13
|
import { NotionMediaDownloader } from './notion-media.js';
|
|
14
14
|
import { parseDatabaseSchema, parseEntryValues, renderPropertiesSection, } from './notion-databases.js';
|
|
15
15
|
import { resolveNotionReferences } from './notion-references.js';
|
|
16
|
-
const MAX_DOCUMENT_COUNT =
|
|
16
|
+
const MAX_DOCUMENT_COUNT = 10_000;
|
|
17
17
|
// ============================================
|
|
18
18
|
// Source
|
|
19
19
|
// ============================================
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { discoverOneNotePages, slugifyPathSegment } from '../onenote-tree.js';
|
|
3
|
+
/** Minimal OneNoteApiClient: one notebook, no groups, pages routed by section. */
|
|
4
|
+
function mockClient(sectionPages) {
|
|
5
|
+
const sections = Object.keys(sectionPages).map((id) => ({ id, displayName: id }));
|
|
6
|
+
return {
|
|
7
|
+
listNotebooks: async () => [{ id: 'nb1', displayName: 'Notebook' }],
|
|
8
|
+
listSectionGroups: async () => [],
|
|
9
|
+
listSectionsInNotebook: async () => sections,
|
|
10
|
+
listSectionsInGroup: async () => [],
|
|
11
|
+
listPages: async (sectionId) => sectionPages[sectionId] ?? [],
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
describe('discoverOneNotePages — dedup same-titled sibling pages (finding D, #203)', () => {
|
|
15
|
+
it('two same-titled pages in ONE section get distinct paths + disambiguated titles', async () => {
|
|
16
|
+
const out = await discoverOneNotePages(mockClient({
|
|
17
|
+
sec1: [
|
|
18
|
+
{ id: 'p1', title: 'Notes' },
|
|
19
|
+
{ id: 'p2', title: 'Notes' },
|
|
20
|
+
],
|
|
21
|
+
}));
|
|
22
|
+
expect(out).toHaveLength(2);
|
|
23
|
+
// Distinct paths — the collision (data loss) is gone.
|
|
24
|
+
expect(new Set(out.map((p) => p.kbPath)).size).toBe(2);
|
|
25
|
+
// Second sibling disambiguated.
|
|
26
|
+
expect(out.map((p) => p.title).sort()).toEqual(['Notes', 'Notes 2']);
|
|
27
|
+
// The server's bijection invariant: pathTail === slug(name).
|
|
28
|
+
for (const p of out) {
|
|
29
|
+
expect(p.kbPath.split('/').pop()).toBe(slugifyPathSegment(p.title));
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
it('same title in DIFFERENT sections is NOT deduped (counter is per-section)', async () => {
|
|
33
|
+
const out = await discoverOneNotePages(mockClient({
|
|
34
|
+
secA: [{ id: 'p1', title: 'Notes' }],
|
|
35
|
+
secB: [{ id: 'p2', title: 'Notes' }],
|
|
36
|
+
}));
|
|
37
|
+
expect(out).toHaveLength(2);
|
|
38
|
+
// Both keep the plain title (no cross-section suffixing)...
|
|
39
|
+
expect(out.map((p) => p.title)).toEqual(['Notes', 'Notes']);
|
|
40
|
+
// ...and the section segment already makes the paths distinct.
|
|
41
|
+
expect(new Set(out.map((p) => p.kbPath)).size).toBe(2);
|
|
42
|
+
});
|
|
43
|
+
});
|
|
@@ -6,7 +6,6 @@
|
|
|
6
6
|
* are simple enough that a hand-rolled client is cheaper than the SDK and
|
|
7
7
|
* gives us precise control over 429 handling.
|
|
8
8
|
*/
|
|
9
|
-
/// <reference types="node" resolution-mode="require"/>
|
|
10
9
|
import type { OneNoteNotebook, OneNotePage, OneNoteSection, OneNoteSectionGroup } from './types.js';
|
|
11
10
|
export interface OneNoteApiClientOptions {
|
|
12
11
|
/** Alternate base URL — for testing only. */
|
package/package.json
CHANGED