@moxn/kb-migrate 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +14 -1
- package/dist/client.js +27 -1
- package/dist/date-filter.d.ts +31 -0
- package/dist/date-filter.js +83 -0
- package/dist/export-notion.d.ts +22 -0
- package/dist/export-notion.js +188 -0
- package/dist/export.js +14 -1
- package/dist/index.js +218 -18
- package/dist/sources/local.d.ts +3 -0
- package/dist/sources/local.js +31 -1
- package/dist/sources/notion-blocks.d.ts +2 -0
- package/dist/sources/notion-blocks.js +52 -24
- package/dist/sources/notion-databases.js +4 -0
- package/dist/sources/notion-media.js +1 -1
- package/dist/sources/notion.d.ts +11 -0
- package/dist/sources/notion.js +44 -3
- package/dist/targets/base.d.ts +77 -0
- package/dist/targets/base.js +21 -0
- package/dist/targets/index.d.ts +5 -0
- package/dist/targets/index.js +5 -0
- package/dist/targets/notion.d.ts +93 -0
- package/dist/targets/notion.js +478 -0
- package/dist/types.d.ts +18 -2
- package/package.json +23 -1
package/dist/index.js
CHANGED
|
@@ -18,6 +18,8 @@ import { NotionSource } from './sources/notion.js';
|
|
|
18
18
|
import { notionColorToHex } from './sources/notion-api.js';
|
|
19
19
|
import { MoxnClient } from './client.js';
|
|
20
20
|
import { runExport } from './export.js';
|
|
21
|
+
import { runNotionExport } from './export-notion.js';
|
|
22
|
+
import { buildDateFilter } from './date-filter.js';
|
|
21
23
|
const DEFAULT_API_URL = 'https://moxn.dev';
|
|
22
24
|
const DEFAULT_EXTENSIONS = ['.md', '.txt'];
|
|
23
25
|
async function runMigration(source, options) {
|
|
@@ -115,6 +117,29 @@ function printExportSummary(log) {
|
|
|
115
117
|
console.log('\n(Dry run - no changes made)');
|
|
116
118
|
}
|
|
117
119
|
}
|
|
120
|
+
function printNotionExportSummary(log) {
|
|
121
|
+
console.log('\n--- Export to Notion Summary ---');
|
|
122
|
+
console.log(`Source: ${log.sourceApi}`);
|
|
123
|
+
console.log(`Target: ${log.target.type} (${log.target.location})`);
|
|
124
|
+
console.log(`Base path: ${log.basePath}`);
|
|
125
|
+
console.log(`Conflict strategy: ${log.options.conflictStrategy}`);
|
|
126
|
+
console.log(`Duration: ${(log.summary.duration / 1000).toFixed(1)}s`);
|
|
127
|
+
console.log('');
|
|
128
|
+
console.log(`Total: ${log.summary.total}`);
|
|
129
|
+
console.log(`Created: ${log.summary.created}`);
|
|
130
|
+
console.log(`Updated: ${log.summary.updated}`);
|
|
131
|
+
console.log(`Skipped: ${log.summary.skipped}`);
|
|
132
|
+
console.log(`Failed: ${log.summary.failed}`);
|
|
133
|
+
if (log.summary.failed > 0) {
|
|
134
|
+
console.log('\nFailed documents:');
|
|
135
|
+
for (const f of log.results.filter((r) => r.status === 'failed')) {
|
|
136
|
+
console.log(` - ${f.documentPath}: ${f.error}`);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (log.options.dryRun) {
|
|
140
|
+
console.log('\n(Dry run - no changes made)');
|
|
141
|
+
}
|
|
142
|
+
}
|
|
118
143
|
const program = new Command();
|
|
119
144
|
program
|
|
120
145
|
.name('moxn-kb-migrate')
|
|
@@ -130,6 +155,10 @@ program
|
|
|
130
155
|
.option('--on-conflict <action>', 'Action on conflict: skip or update', 'skip')
|
|
131
156
|
.option('--default-permission <perm>', 'Default permission: edit, read, or none')
|
|
132
157
|
.option('--ai-access <perm>', 'AI access permission: edit, read, or none')
|
|
158
|
+
.option('--created-after <date>', 'Only include docs created after this date (ISO 8601)')
|
|
159
|
+
.option('--created-before <date>', 'Only include docs created before this date (ISO 8601)')
|
|
160
|
+
.option('--modified-after <date>', 'Only include docs modified after this date (ISO 8601)')
|
|
161
|
+
.option('--modified-before <date>', 'Only include docs modified before this date (ISO 8601)')
|
|
133
162
|
.option('--dry-run', 'Preview without making changes', false)
|
|
134
163
|
.option('--json', 'Output results as JSON', false)
|
|
135
164
|
.action(async (directory, opts) => {
|
|
@@ -144,9 +173,16 @@ program
|
|
|
144
173
|
console.error('Error: --on-conflict must be "skip" or "update"');
|
|
145
174
|
process.exit(1);
|
|
146
175
|
}
|
|
176
|
+
const dateFilter = buildDateFilter({
|
|
177
|
+
createdAfter: opts.createdAfter,
|
|
178
|
+
createdBefore: opts.createdBefore,
|
|
179
|
+
modifiedAfter: opts.modifiedAfter,
|
|
180
|
+
modifiedBefore: opts.modifiedBefore,
|
|
181
|
+
});
|
|
147
182
|
const source = new LocalSource({
|
|
148
183
|
directory,
|
|
149
184
|
extensions,
|
|
185
|
+
dateFilter,
|
|
150
186
|
});
|
|
151
187
|
const migrationOptions = {
|
|
152
188
|
apiUrl: opts.apiUrl,
|
|
@@ -156,6 +192,7 @@ program
|
|
|
156
192
|
dryRun: opts.dryRun,
|
|
157
193
|
defaultPermission: opts.defaultPermission,
|
|
158
194
|
aiAccess: opts.aiAccess,
|
|
195
|
+
dateFilter,
|
|
159
196
|
};
|
|
160
197
|
try {
|
|
161
198
|
const log = await runMigration(source, migrationOptions);
|
|
@@ -184,6 +221,10 @@ program
|
|
|
184
221
|
.option('--image-dir <name>', 'Directory name for images', 'images')
|
|
185
222
|
.option('--pdf-dir <name>', 'Directory name for PDFs', 'pdfs')
|
|
186
223
|
.option('--csv-dir <name>', 'Directory name for CSVs', 'csvs')
|
|
224
|
+
.option('--created-after <date>', 'Only include docs created after this date (ISO 8601)')
|
|
225
|
+
.option('--created-before <date>', 'Only include docs created before this date (ISO 8601)')
|
|
226
|
+
.option('--modified-after <date>', 'Only include docs modified after this date (ISO 8601)')
|
|
227
|
+
.option('--modified-before <date>', 'Only include docs modified before this date (ISO 8601)')
|
|
187
228
|
.option('--dry-run', 'Preview without writing files', false)
|
|
188
229
|
.option('--json', 'Output results as JSON', false)
|
|
189
230
|
.action(async (directory, opts) => {
|
|
@@ -192,6 +233,12 @@ program
|
|
|
192
233
|
console.error('Error: API key required. Use --api-key or set MOXN_API_KEY env var.');
|
|
193
234
|
process.exit(1);
|
|
194
235
|
}
|
|
236
|
+
const dateFilter = buildDateFilter({
|
|
237
|
+
createdAfter: opts.createdAfter,
|
|
238
|
+
createdBefore: opts.createdBefore,
|
|
239
|
+
modifiedAfter: opts.modifiedAfter,
|
|
240
|
+
modifiedBefore: opts.modifiedBefore,
|
|
241
|
+
});
|
|
195
242
|
const exportOptions = {
|
|
196
243
|
apiUrl: opts.apiUrl,
|
|
197
244
|
apiKey,
|
|
@@ -200,6 +247,7 @@ program
|
|
|
200
247
|
pdfDir: opts.pdfDir,
|
|
201
248
|
csvDir: opts.csvDir,
|
|
202
249
|
dryRun: opts.dryRun,
|
|
250
|
+
dateFilter,
|
|
203
251
|
};
|
|
204
252
|
try {
|
|
205
253
|
const log = await runExport(directory, exportOptions);
|
|
@@ -231,6 +279,10 @@ program
|
|
|
231
279
|
.option('--default-permission <perm>', 'Default permission: edit, read, or none')
|
|
232
280
|
.option('--ai-access <perm>', 'AI access permission: edit, read, or none')
|
|
233
281
|
.option('--visibility <vis>', 'Convenience flag: team (read) or private (none)')
|
|
282
|
+
.option('--created-after <date>', 'Only include docs created after this date (ISO 8601)')
|
|
283
|
+
.option('--created-before <date>', 'Only include docs created before this date (ISO 8601)')
|
|
284
|
+
.option('--modified-after <date>', 'Only include docs modified after this date (ISO 8601)')
|
|
285
|
+
.option('--modified-before <date>', 'Only include docs modified before this date (ISO 8601)')
|
|
234
286
|
.option('--dry-run', 'Preview without making changes', false)
|
|
235
287
|
.option('--json', 'Output results as JSON', false)
|
|
236
288
|
.action(async (opts) => {
|
|
@@ -254,10 +306,17 @@ program
|
|
|
254
306
|
if (!defaultPermission && opts.visibility) {
|
|
255
307
|
defaultPermission = opts.visibility === 'private' ? 'none' : 'read';
|
|
256
308
|
}
|
|
309
|
+
const dateFilter = buildDateFilter({
|
|
310
|
+
createdAfter: opts.createdAfter,
|
|
311
|
+
createdBefore: opts.createdBefore,
|
|
312
|
+
modifiedAfter: opts.modifiedAfter,
|
|
313
|
+
modifiedBefore: opts.modifiedBefore,
|
|
314
|
+
});
|
|
257
315
|
const source = new NotionSource({
|
|
258
316
|
token,
|
|
259
317
|
rootPageId: opts.rootPageId,
|
|
260
318
|
maxDepth: opts.maxDepth ? parseInt(opts.maxDepth, 10) : undefined,
|
|
319
|
+
dateFilter,
|
|
261
320
|
});
|
|
262
321
|
const migrationOptions = {
|
|
263
322
|
apiUrl: opts.apiUrl,
|
|
@@ -268,25 +327,58 @@ program
|
|
|
268
327
|
defaultPermission,
|
|
269
328
|
aiAccess: opts.aiAccess,
|
|
270
329
|
visibility: opts.visibility,
|
|
330
|
+
dateFilter,
|
|
271
331
|
};
|
|
272
332
|
try {
|
|
273
|
-
//
|
|
333
|
+
// Step 1: Validate source (discover pages + databases)
|
|
334
|
+
await source.validate();
|
|
335
|
+
// Step 2: Pre-create databases before document extraction so that
|
|
336
|
+
// child_database blocks can be converted to database_embed blocks.
|
|
337
|
+
const databaseIdMap = new Map();
|
|
338
|
+
const dbImports = source.getDatabaseImports();
|
|
339
|
+
if (!opts.dryRun && dbImports.length > 0) {
|
|
340
|
+
console.log(`\nPre-creating ${dbImports.length} database(s)...`);
|
|
341
|
+
const preClient = new MoxnClient(migrationOptions);
|
|
342
|
+
for (const dbImport of dbImports) {
|
|
343
|
+
try {
|
|
344
|
+
const kbDbId = await preCreateNotionDatabase(preClient, dbImport);
|
|
345
|
+
databaseIdMap.set(dbImport.notionDatabaseId, kbDbId);
|
|
346
|
+
}
|
|
347
|
+
catch (error) {
|
|
348
|
+
console.error(` Error pre-creating database "${dbImport.schema.name}": ${error instanceof Error ? error.message : error}`);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
// Make the map available during extraction
|
|
352
|
+
source.setDatabaseIdMap(databaseIdMap);
|
|
353
|
+
console.log(` ${databaseIdMap.size} database(s) pre-created for embed resolution.\n`);
|
|
354
|
+
}
|
|
355
|
+
// Step 3: Run page migration (validate is idempotent, extract uses databaseIdMap)
|
|
274
356
|
const log = await runMigration(source, migrationOptions);
|
|
275
|
-
//
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
357
|
+
// Step 4: Finalize databases — create columns, link entries, assign tags.
|
|
358
|
+
// Database records already exist from pre-create; this step adds schema + data.
|
|
359
|
+
if (!opts.dryRun && dbImports.length > 0) {
|
|
360
|
+
console.log(`\nFinalizing ${dbImports.length} database(s)...`);
|
|
361
|
+
const client = new MoxnClient(migrationOptions);
|
|
362
|
+
let totalEntries = 0;
|
|
363
|
+
const allPropertyTypes = new Set();
|
|
364
|
+
for (const dbImport of dbImports) {
|
|
365
|
+
try {
|
|
366
|
+
const preCreatedId = databaseIdMap.get(dbImport.notionDatabaseId);
|
|
367
|
+
await importNotionDatabase(client, dbImport, log, migrationOptions, preCreatedId);
|
|
368
|
+
totalEntries += dbImport.entries.length;
|
|
369
|
+
// Track property types encountered
|
|
370
|
+
for (const col of dbImport.schema.mappedColumns) {
|
|
371
|
+
allPropertyTypes.add(col.notionType);
|
|
284
372
|
}
|
|
285
|
-
|
|
286
|
-
|
|
373
|
+
for (const col of dbImport.schema.unmappedColumns) {
|
|
374
|
+
allPropertyTypes.add(col.notionType);
|
|
287
375
|
}
|
|
288
376
|
}
|
|
377
|
+
catch (error) {
|
|
378
|
+
console.error(` Error finalizing database "${dbImport.schema.name}": ${error instanceof Error ? error.message : error}`);
|
|
379
|
+
}
|
|
289
380
|
}
|
|
381
|
+
console.log(`\nImported ${dbImports.length} databases, ${totalEntries} entries, ${allPropertyTypes.size} property types`);
|
|
290
382
|
}
|
|
291
383
|
// Cleanup temp files
|
|
292
384
|
await source.cleanup();
|
|
@@ -306,20 +398,128 @@ program
|
|
|
306
398
|
process.exit(1);
|
|
307
399
|
}
|
|
308
400
|
});
|
|
401
|
+
program
|
|
402
|
+
.command('export-notion')
|
|
403
|
+
.description('Export documents from Moxn Knowledge Base to Notion')
|
|
404
|
+
.option('--api-key <key>', 'Moxn API key (or set MOXN_API_KEY env var)')
|
|
405
|
+
.option('--api-url <url>', 'Moxn API base URL', DEFAULT_API_URL)
|
|
406
|
+
.option('--notion-token <token>', 'Notion integration token (or set NOTION_TOKEN env var)')
|
|
407
|
+
.option('--parent-page-id <id>', 'Notion parent page under which to create pages')
|
|
408
|
+
.option('--base-path <path>', 'Only export docs under this path prefix', '/')
|
|
409
|
+
.option('--conflict-strategy <strategy>', 'How to handle existing pages: skip or update', 'skip')
|
|
410
|
+
.option('--modified-after <date>', 'Only include docs modified after this date (ISO 8601)')
|
|
411
|
+
.option('--modified-before <date>', 'Only include docs modified before this date (ISO 8601)')
|
|
412
|
+
.option('--created-after <date>', 'Only include docs created after this date (ISO 8601)')
|
|
413
|
+
.option('--created-before <date>', 'Only include docs created before this date (ISO 8601)')
|
|
414
|
+
.option('--dry-run', 'Preview without making changes', false)
|
|
415
|
+
.option('--json', 'Output results as JSON', false)
|
|
416
|
+
.action(async (opts) => {
|
|
417
|
+
const apiKey = opts.apiKey || process.env.MOXN_API_KEY;
|
|
418
|
+
if (!apiKey) {
|
|
419
|
+
console.error('Error: Moxn API key required. Use --api-key or set MOXN_API_KEY env var.');
|
|
420
|
+
process.exit(1);
|
|
421
|
+
}
|
|
422
|
+
const notionToken = opts.notionToken || process.env.NOTION_TOKEN;
|
|
423
|
+
if (!notionToken) {
|
|
424
|
+
console.error('Error: Notion token required. Use --notion-token or set NOTION_TOKEN env var.');
|
|
425
|
+
process.exit(1);
|
|
426
|
+
}
|
|
427
|
+
if (!opts.parentPageId) {
|
|
428
|
+
console.error('Error: --parent-page-id is required.');
|
|
429
|
+
process.exit(1);
|
|
430
|
+
}
|
|
431
|
+
const conflictStrategy = opts.conflictStrategy;
|
|
432
|
+
if (!['skip', 'update'].includes(conflictStrategy)) {
|
|
433
|
+
console.error('Error: --conflict-strategy must be "skip" or "update"');
|
|
434
|
+
process.exit(1);
|
|
435
|
+
}
|
|
436
|
+
const dateFilter = buildDateFilter({
|
|
437
|
+
createdAfter: opts.createdAfter,
|
|
438
|
+
createdBefore: opts.createdBefore,
|
|
439
|
+
modifiedAfter: opts.modifiedAfter,
|
|
440
|
+
modifiedBefore: opts.modifiedBefore,
|
|
441
|
+
});
|
|
442
|
+
try {
|
|
443
|
+
const log = await runNotionExport({
|
|
444
|
+
apiUrl: opts.apiUrl,
|
|
445
|
+
apiKey,
|
|
446
|
+
notionToken,
|
|
447
|
+
parentPageId: opts.parentPageId,
|
|
448
|
+
basePath: opts.basePath,
|
|
449
|
+
conflictStrategy,
|
|
450
|
+
dryRun: opts.dryRun,
|
|
451
|
+
dateFilter,
|
|
452
|
+
});
|
|
453
|
+
if (opts.json) {
|
|
454
|
+
console.log(JSON.stringify(log, null, 2));
|
|
455
|
+
}
|
|
456
|
+
else {
|
|
457
|
+
printNotionExportSummary(log);
|
|
458
|
+
}
|
|
459
|
+
if (log.summary.failed > 0) {
|
|
460
|
+
process.exit(1);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
catch (error) {
|
|
464
|
+
console.error('Export to Notion failed:', error instanceof Error ? error.message : error);
|
|
465
|
+
process.exit(1);
|
|
466
|
+
}
|
|
467
|
+
});
|
|
309
468
|
/**
|
|
310
469
|
* Import a Notion database into Moxn KB.
|
|
311
470
|
*
|
|
312
471
|
* Creates the database, columns with tags, links entries, and assigns tag values.
|
|
313
472
|
*/
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
473
|
+
/**
|
|
474
|
+
* Pre-create a KB database and store its Notion mapping.
|
|
475
|
+
* Called before document extraction so child_database blocks can resolve.
|
|
476
|
+
* Returns the KB database ID.
|
|
477
|
+
*/
|
|
478
|
+
async function preCreateNotionDatabase(client, dbImport) {
|
|
479
|
+
const { schema } = dbImport;
|
|
480
|
+
console.log(` Pre-creating database: ${schema.name}`);
|
|
318
481
|
const db = await client.createDatabase({
|
|
319
482
|
name: schema.name,
|
|
320
483
|
description: schema.description || undefined,
|
|
321
484
|
});
|
|
322
485
|
console.log(` Database created: ${db.id}`);
|
|
486
|
+
// Store Notion → KB database mapping
|
|
487
|
+
try {
|
|
488
|
+
await client.createNotionDatabaseMapping({
|
|
489
|
+
kbDatabaseId: db.id,
|
|
490
|
+
notionDatabaseId: dbImport.notionDatabaseId,
|
|
491
|
+
notionDatabaseTitle: schema.name,
|
|
492
|
+
});
|
|
493
|
+
console.log(` Database mapping stored: ${dbImport.notionDatabaseId} → ${db.id}`);
|
|
494
|
+
}
|
|
495
|
+
catch (error) {
|
|
496
|
+
console.warn(` Warning: Failed to store database mapping: ${error instanceof Error ? error.message : error}`);
|
|
497
|
+
}
|
|
498
|
+
return db.id;
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Import a Notion database into Moxn KB.
|
|
502
|
+
*
|
|
503
|
+
* Creates columns with tags, links entries, and assigns tag values.
|
|
504
|
+
* If preCreatedDbId is provided, skips database creation (already done in pre-create step).
|
|
505
|
+
*/
|
|
506
|
+
async function importNotionDatabase(client, dbImport, log, options, preCreatedDbId) {
|
|
507
|
+
const { schema, entries } = dbImport;
|
|
508
|
+
let dbId;
|
|
509
|
+
if (preCreatedDbId) {
|
|
510
|
+
dbId = preCreatedDbId;
|
|
511
|
+
console.log(` Finalizing database: ${schema.name} (${dbId})`);
|
|
512
|
+
}
|
|
513
|
+
else {
|
|
514
|
+
// Fallback: create database now (non-Notion sources or if pre-create was skipped)
|
|
515
|
+
console.log(` Creating database: ${schema.name}`);
|
|
516
|
+
const db = await client.createDatabase({
|
|
517
|
+
name: schema.name,
|
|
518
|
+
description: schema.description || undefined,
|
|
519
|
+
});
|
|
520
|
+
dbId = db.id;
|
|
521
|
+
console.log(` Database created: ${dbId}`);
|
|
522
|
+
}
|
|
323
523
|
// 2. Create columns with tags
|
|
324
524
|
// Map: column name → { columnId, optionTagMap: option name → tagId }
|
|
325
525
|
const columnMap = new Map();
|
|
@@ -344,7 +544,7 @@ async function importNotionDatabase(client, dbImport, log, options) {
|
|
|
344
544
|
}
|
|
345
545
|
// Create the column
|
|
346
546
|
try {
|
|
347
|
-
const column = await client.addDatabaseColumn(
|
|
547
|
+
const column = await client.addDatabaseColumn(dbId, {
|
|
348
548
|
name: col.notionPropertyName,
|
|
349
549
|
type: col.moxnType,
|
|
350
550
|
optionTagIds: tagIds,
|
|
@@ -384,7 +584,7 @@ async function importNotionDatabase(client, dbImport, log, options) {
|
|
|
384
584
|
}
|
|
385
585
|
// Add document to database
|
|
386
586
|
try {
|
|
387
|
-
await client.addDocumentToDatabase(
|
|
587
|
+
await client.addDocumentToDatabase(dbId, docInfo.documentId);
|
|
388
588
|
linkedCount++;
|
|
389
589
|
// Parse and assign tag values
|
|
390
590
|
const { parseEntryValues } = await import('./sources/notion-databases.js');
|
package/dist/sources/local.d.ts
CHANGED
|
@@ -5,11 +5,14 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import { MigrationSource, type SourceConfig } from './base.js';
|
|
7
7
|
import type { ExtractedDocument } from '../types.js';
|
|
8
|
+
import type { DateFilter } from '../date-filter.js';
|
|
8
9
|
export interface LocalSourceConfig extends SourceConfig {
|
|
9
10
|
/** Directory path to scan for documents */
|
|
10
11
|
directory: string;
|
|
11
12
|
/** File extensions to include (default: ['.md', '.txt']) */
|
|
12
13
|
extensions: string[];
|
|
14
|
+
/** Date filter for source files */
|
|
15
|
+
dateFilter?: DateFilter;
|
|
13
16
|
}
|
|
14
17
|
/**
|
|
15
18
|
* Local filesystem migration source
|
package/dist/sources/local.js
CHANGED
|
@@ -60,7 +60,37 @@ export class LocalSource extends MigrationSource {
|
|
|
60
60
|
allFiles.push(...matches);
|
|
61
61
|
}
|
|
62
62
|
// Deduplicate and sort
|
|
63
|
-
|
|
63
|
+
let uniqueFiles = [...new Set(allFiles)].sort();
|
|
64
|
+
// Apply date filter if configured
|
|
65
|
+
if (this.config.dateFilter) {
|
|
66
|
+
const { matchesDateFilter } = await import('../date-filter.js');
|
|
67
|
+
const filtered = [];
|
|
68
|
+
let birthtimeWarned = false;
|
|
69
|
+
for (const file of uniqueFiles) {
|
|
70
|
+
const fullPath = path.join(this.config.directory, file);
|
|
71
|
+
const stat = await fs.stat(fullPath);
|
|
72
|
+
// Warn if birthtime is unreliable (equals ctime on some systems)
|
|
73
|
+
if (!birthtimeWarned &&
|
|
74
|
+
(this.config.dateFilter.createdAfter ||
|
|
75
|
+
this.config.dateFilter.createdBefore) &&
|
|
76
|
+
stat.birthtime.getTime() === stat.ctime.getTime()) {
|
|
77
|
+
console.warn(' \u26a0 File creation time (birthtime) may be unreliable on this filesystem.');
|
|
78
|
+
console.warn(' --created-after/--created-before results may be inaccurate.');
|
|
79
|
+
birthtimeWarned = true;
|
|
80
|
+
}
|
|
81
|
+
if (matchesDateFilter(this.config.dateFilter, {
|
|
82
|
+
createdAt: stat.birthtime,
|
|
83
|
+
modifiedAt: stat.mtime,
|
|
84
|
+
})) {
|
|
85
|
+
filtered.push(file);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const skipped = uniqueFiles.length - filtered.length;
|
|
89
|
+
if (skipped > 0) {
|
|
90
|
+
console.log(` Filtered ${skipped} files by date criteria`);
|
|
91
|
+
}
|
|
92
|
+
uniqueFiles = filtered;
|
|
93
|
+
}
|
|
64
94
|
// Detect KB path collisions (e.g., doc.md and doc.mdx both map to /doc)
|
|
65
95
|
const pathToFiles = new Map();
|
|
66
96
|
for (const file of uniqueFiles) {
|
|
@@ -17,6 +17,8 @@ export type PagePathMap = Map<string, string>;
|
|
|
17
17
|
export declare function blocksToSections(blocks: NotionBlock[], client: NotionApiClient, pagePathMap: PagePathMap, options?: {
|
|
18
18
|
/** Track synced block IDs to detect cycles. */
|
|
19
19
|
visitedSyncedBlocks?: Set<string>;
|
|
20
|
+
/** Map of Notion database IDs to KB database IDs (for child_database blocks). */
|
|
21
|
+
databaseIdMap?: Map<string, string>;
|
|
20
22
|
}): Promise<SectionInput[]>;
|
|
21
23
|
/** Convert rich text array to markdown string. */
|
|
22
24
|
export declare function richTextToMarkdown(richText: NotionRichText[]): string;
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
17
17
|
const visitedSyncedBlocks = options?.visitedSyncedBlocks ?? new Set();
|
|
18
|
+
const databaseIdMap = options?.databaseIdMap;
|
|
18
19
|
const sections = [];
|
|
19
20
|
let currentSectionName = 'Introduction';
|
|
20
21
|
let currentBlocks = [];
|
|
@@ -23,7 +24,10 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
|
23
24
|
if (block.type === 'heading_2') {
|
|
24
25
|
// Flush current section if it has content
|
|
25
26
|
if (currentBlocks.length > 0) {
|
|
26
|
-
sections.push({
|
|
27
|
+
sections.push({
|
|
28
|
+
name: currentSectionName,
|
|
29
|
+
content: mergeConsecutiveListBlocks(currentBlocks),
|
|
30
|
+
});
|
|
27
31
|
}
|
|
28
32
|
const h2 = block;
|
|
29
33
|
currentSectionName = richTextToPlain(h2.heading_2.rich_text) || 'Untitled';
|
|
@@ -31,12 +35,15 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
|
31
35
|
continue;
|
|
32
36
|
}
|
|
33
37
|
// Convert block to content blocks
|
|
34
|
-
const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks);
|
|
38
|
+
const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
|
|
35
39
|
currentBlocks.push(...converted);
|
|
36
40
|
}
|
|
37
41
|
// Flush last section
|
|
38
42
|
if (currentBlocks.length > 0) {
|
|
39
|
-
sections.push({
|
|
43
|
+
sections.push({
|
|
44
|
+
name: currentSectionName,
|
|
45
|
+
content: mergeConsecutiveListBlocks(currentBlocks),
|
|
46
|
+
});
|
|
40
47
|
}
|
|
41
48
|
// If no sections were created at all (empty page), return empty array
|
|
42
49
|
return sections;
|
|
@@ -44,7 +51,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
|
44
51
|
// ============================================
|
|
45
52
|
// Block conversion
|
|
46
53
|
// ============================================
|
|
47
|
-
async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
54
|
+
async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
48
55
|
const results = [];
|
|
49
56
|
switch (block.type) {
|
|
50
57
|
case 'paragraph':
|
|
@@ -74,10 +81,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
|
74
81
|
results.push(...convertToDo(block));
|
|
75
82
|
break;
|
|
76
83
|
case 'quote':
|
|
77
|
-
results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks)));
|
|
84
|
+
results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
78
85
|
break;
|
|
79
86
|
case 'callout':
|
|
80
|
-
results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks)));
|
|
87
|
+
results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
81
88
|
break;
|
|
82
89
|
case 'divider':
|
|
83
90
|
results.push(textBlock('---'));
|
|
@@ -86,7 +93,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
|
86
93
|
results.push(...(await convertTable(block, client)));
|
|
87
94
|
break;
|
|
88
95
|
case 'toggle':
|
|
89
|
-
results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks)));
|
|
96
|
+
results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
90
97
|
break;
|
|
91
98
|
case 'bookmark':
|
|
92
99
|
results.push(...convertBookmark(block));
|
|
@@ -109,13 +116,13 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
|
109
116
|
// Just note it in the content.
|
|
110
117
|
break;
|
|
111
118
|
case 'child_database':
|
|
112
|
-
|
|
119
|
+
results.push(...convertChildDatabase(block, databaseIdMap));
|
|
113
120
|
break;
|
|
114
121
|
case 'synced_block':
|
|
115
|
-
results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks)));
|
|
122
|
+
results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
116
123
|
break;
|
|
117
124
|
case 'column_list':
|
|
118
|
-
results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks)));
|
|
125
|
+
results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
119
126
|
break;
|
|
120
127
|
case 'column':
|
|
121
128
|
// Columns are handled by column_list
|
|
@@ -142,12 +149,20 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
|
142
149
|
}
|
|
143
150
|
// If block has children (except types that handle their own)
|
|
144
151
|
if (block.has_children &&
|
|
145
|
-
![
|
|
152
|
+
![
|
|
153
|
+
'table',
|
|
154
|
+
'toggle',
|
|
155
|
+
'synced_block',
|
|
156
|
+
'column_list',
|
|
157
|
+
'column',
|
|
158
|
+
'quote',
|
|
159
|
+
'callout',
|
|
160
|
+
].includes(block.type)) {
|
|
146
161
|
const indent = ['bulleted_list_item', 'numbered_list_item', 'to_do'].includes(block.type)
|
|
147
162
|
? ' '
|
|
148
163
|
: undefined;
|
|
149
164
|
const children = await client.getBlockChildren(block.id);
|
|
150
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent);
|
|
165
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap);
|
|
151
166
|
results.push(...childBlocks);
|
|
152
167
|
}
|
|
153
168
|
return results;
|
|
@@ -193,7 +208,7 @@ function convertToDo(block) {
|
|
|
193
208
|
const text = richTextToMarkdown(td.to_do.rich_text);
|
|
194
209
|
return [textBlock(`${checkbox} ${text}`)];
|
|
195
210
|
}
|
|
196
|
-
async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
211
|
+
async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
197
212
|
const q = block;
|
|
198
213
|
const text = richTextToMarkdown(q.quote.rich_text);
|
|
199
214
|
if (!text && !block.has_children)
|
|
@@ -207,12 +222,12 @@ async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
|
207
222
|
const results = quoted ? [textBlock(quoted)] : [];
|
|
208
223
|
if (block.has_children) {
|
|
209
224
|
const children = await client.getBlockChildren(block.id);
|
|
210
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ');
|
|
225
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
211
226
|
results.push(...childBlocks);
|
|
212
227
|
}
|
|
213
228
|
return results;
|
|
214
229
|
}
|
|
215
|
-
async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
230
|
+
async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
216
231
|
const c = block;
|
|
217
232
|
const text = richTextToMarkdown(c.callout.rich_text);
|
|
218
233
|
const emoji = c.callout.icon?.emoji ?? '';
|
|
@@ -224,7 +239,7 @@ async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
|
224
239
|
const results = [textBlock(quoted)];
|
|
225
240
|
if (block.has_children) {
|
|
226
241
|
const children = await client.getBlockChildren(block.id);
|
|
227
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ');
|
|
242
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
228
243
|
results.push(...childBlocks);
|
|
229
244
|
}
|
|
230
245
|
return results;
|
|
@@ -253,17 +268,30 @@ async function convertTable(block, client) {
|
|
|
253
268
|
}
|
|
254
269
|
return [textBlock(lines.join('\n'))];
|
|
255
270
|
}
|
|
256
|
-
async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
271
|
+
async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
257
272
|
const t = block;
|
|
258
273
|
const header = richTextToMarkdown(t.toggle.rich_text);
|
|
259
274
|
const results = [textBlock(`**${header}**`)];
|
|
260
275
|
if (block.has_children) {
|
|
261
276
|
const children = await client.getBlockChildren(block.id);
|
|
262
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ');
|
|
277
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
263
278
|
results.push(...childBlocks);
|
|
264
279
|
}
|
|
265
280
|
return results;
|
|
266
281
|
}
|
|
282
|
+
function convertChildDatabase(block, databaseIdMap) {
|
|
283
|
+
const cd = block;
|
|
284
|
+
const title = cd.child_database?.title || 'Untitled Database';
|
|
285
|
+
const notionDbId = normalizeId(block.id);
|
|
286
|
+
// Look up KB database ID from mapping
|
|
287
|
+
const kbDatabaseId = databaseIdMap?.get(notionDbId);
|
|
288
|
+
if (kbDatabaseId) {
|
|
289
|
+
return [{ blockType: 'database_embed', databaseId: kbDatabaseId }];
|
|
290
|
+
}
|
|
291
|
+
// No mapping available — emit text placeholder
|
|
292
|
+
console.warn(` No KB mapping for Notion database "${title}" (${notionDbId})`);
|
|
293
|
+
return [textBlock(`*(Embedded database: ${title})*`)];
|
|
294
|
+
}
|
|
267
295
|
function convertBookmark(block) {
|
|
268
296
|
const b = block;
|
|
269
297
|
const url = b.bookmark.url;
|
|
@@ -354,7 +382,7 @@ function convertLinkToPage(block, pagePathMap) {
|
|
|
354
382
|
// Target not in import set
|
|
355
383
|
return [textBlock(`*(Link to Notion page: ${targetId})*`)];
|
|
356
384
|
}
|
|
357
|
-
async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
385
|
+
async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
358
386
|
const sb = block;
|
|
359
387
|
// Get the source block ID (either this block or the original)
|
|
360
388
|
const sourceId = sb.synced_block.synced_from?.block_id ?? block.id;
|
|
@@ -365,19 +393,19 @@ async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlock
|
|
|
365
393
|
visitedSyncedBlocks.add(sourceId);
|
|
366
394
|
try {
|
|
367
395
|
const children = await client.getBlockChildren(sourceId);
|
|
368
|
-
return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks);
|
|
396
|
+
return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
|
|
369
397
|
}
|
|
370
398
|
finally {
|
|
371
399
|
visitedSyncedBlocks.delete(sourceId);
|
|
372
400
|
}
|
|
373
401
|
}
|
|
374
|
-
async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks) {
|
|
402
|
+
async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
375
403
|
const children = await client.getBlockChildren(block.id);
|
|
376
404
|
const results = [];
|
|
377
405
|
for (const column of children) {
|
|
378
406
|
if (column.type === 'column' && column.has_children) {
|
|
379
407
|
const columnChildren = await client.getBlockChildren(column.id);
|
|
380
|
-
const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks);
|
|
408
|
+
const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
|
|
381
409
|
results.push(...converted);
|
|
382
410
|
}
|
|
383
411
|
}
|
|
@@ -526,10 +554,10 @@ function mergeConsecutiveListBlocks(blocks) {
|
|
|
526
554
|
/**
|
|
527
555
|
* Convert children blocks, merge consecutive list items, and optionally indent.
|
|
528
556
|
*/
|
|
529
|
-
async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent) {
|
|
557
|
+
async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap) {
|
|
530
558
|
const blocks = [];
|
|
531
559
|
for (const child of children) {
|
|
532
|
-
const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks);
|
|
560
|
+
const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
|
|
533
561
|
blocks.push(...converted);
|
|
534
562
|
}
|
|
535
563
|
const merged = mergeConsecutiveListBlocks(blocks);
|
|
@@ -63,6 +63,10 @@ export function parseDatabaseSchema(db) {
|
|
|
63
63
|
});
|
|
64
64
|
}
|
|
65
65
|
else {
|
|
66
|
+
// Warn on property types that have no meaningful static value
|
|
67
|
+
if (prop.type === 'button' || prop.type === 'ai_text') {
|
|
68
|
+
console.warn(` Warning: Skipping unsupported property "${propName}" (type: ${prop.type})`);
|
|
69
|
+
}
|
|
66
70
|
unmappedColumns.push({
|
|
67
71
|
notionPropertyId: prop.id,
|
|
68
72
|
notionPropertyName: propName,
|
|
@@ -82,7 +82,7 @@ export class NotionMediaDownloader {
|
|
|
82
82
|
}
|
|
83
83
|
/** Check if a content block has a Notion signed URL that needs downloading. Returns typed block or null. */
|
|
84
84
|
asNotionSignedUrlBlock(block) {
|
|
85
|
-
if (block.blockType === 'text')
|
|
85
|
+
if (block.blockType === 'text' || block.blockType === 'database_embed')
|
|
86
86
|
return null;
|
|
87
87
|
if (block.type !== 'url' || !block.url)
|
|
88
88
|
return null;
|