@clarionstack/notesia 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/notesia.mjs +13 -0
- package/package.json +1 -1
- package/src/import.mjs +150 -70
package/bin/notesia.mjs
CHANGED
|
@@ -1,9 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { createRequire } from 'module';
|
|
3
|
+
import { fileURLToPath } from 'url';
|
|
4
|
+
import { dirname, resolve } from 'path';
|
|
2
5
|
import { runCli } from '../src/commands.mjs';
|
|
3
6
|
import { runImport } from '../src/import.mjs';
|
|
4
7
|
|
|
8
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
9
|
+
const require = createRequire(import.meta.url);
|
|
10
|
+
|
|
5
11
|
const argv = process.argv.slice(2);
|
|
6
12
|
|
|
13
|
+
// Handle version flag / command
|
|
14
|
+
if (argv[0] === 'version' || argv[0] === '-v' || argv[0] === '--version') {
|
|
15
|
+
const { version } = require(resolve(__dirname, '../package.json'));
|
|
16
|
+
process.stdout.write(`notesia v${version}\n`);
|
|
17
|
+
process.exit(0);
|
|
18
|
+
}
|
|
19
|
+
|
|
7
20
|
// Dispatch `notesia import` to the dedicated import handler
|
|
8
21
|
if (argv[0] === 'import') {
|
|
9
22
|
runImport(argv.slice(1)).catch((error) => {
|
package/package.json
CHANGED
package/src/import.mjs
CHANGED
|
@@ -12,6 +12,8 @@ import readline from 'node:readline';
|
|
|
12
12
|
import { requestBridge } from './client.mjs';
|
|
13
13
|
|
|
14
14
|
const NOTE_CHAR_LIMIT = 100_000;
|
|
15
|
+
const BRIDGE_MAX_REQUEST_BYTES = 1024 * 1024;
|
|
16
|
+
const BRIDGE_TARGET_REQUEST_BYTES = Math.floor(BRIDGE_MAX_REQUEST_BYTES * 0.9);
|
|
15
17
|
const DEFAULT_EXTENSIONS = ['md', 'txt', 'pdf', 'docx', 'json'];
|
|
16
18
|
const JSON_MAX_DEPTH = 12;
|
|
17
19
|
const JSON_MAX_ENTRIES = 5_000;
|
|
@@ -514,7 +516,71 @@ function splitContent(title, body) {
|
|
|
514
516
|
chunks.push(remaining.slice(0, splitAt));
|
|
515
517
|
remaining = remaining.slice(splitAt);
|
|
516
518
|
}
|
|
517
|
-
return chunks.map((content, i) => ({
|
|
519
|
+
return chunks.map((content, i) => ({
|
|
520
|
+
title: i === 0 ? title : `${title}_${i + 1}`,
|
|
521
|
+
content,
|
|
522
|
+
}));
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function normalizeImportItem(item) {
|
|
526
|
+
return {
|
|
527
|
+
title: item.title,
|
|
528
|
+
content: item._parseError ? '' : item.content,
|
|
529
|
+
source_path: item.source_path,
|
|
530
|
+
_parseError: item._parseError || null,
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function buildBulkImportPayload(items, opts, createFolderIfMissing = false) {
|
|
535
|
+
return {
|
|
536
|
+
items: items.map(normalizeImportItem),
|
|
537
|
+
folder: opts.folder || null,
|
|
538
|
+
tags: opts.tags || [],
|
|
539
|
+
conflict: opts.conflict || 'skip',
|
|
540
|
+
create_folder_if_missing: createFolderIfMissing,
|
|
541
|
+
dry_run: opts.dryRun,
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function estimateBridgeRequestBytes(payload) {
|
|
546
|
+
return Buffer.byteLength(JSON.stringify({ op: 'bulk_import', payload }), 'utf8');
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
function splitItemsIntoPayloadBatches(items, opts) {
|
|
550
|
+
if (items.length === 0) return [[]];
|
|
551
|
+
|
|
552
|
+
const batches = [];
|
|
553
|
+
let current = [];
|
|
554
|
+
|
|
555
|
+
for (const item of items) {
|
|
556
|
+
const candidate = [...current, item];
|
|
557
|
+
const candidatePayload = buildBulkImportPayload(candidate, opts, false);
|
|
558
|
+
const candidateBytes = estimateBridgeRequestBytes(candidatePayload);
|
|
559
|
+
|
|
560
|
+
if (current.length > 0 && candidateBytes > BRIDGE_TARGET_REQUEST_BYTES) {
|
|
561
|
+
batches.push(current);
|
|
562
|
+
current = [item];
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
current = candidate;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
if (current.length > 0) {
|
|
570
|
+
batches.push(current);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
return batches;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
function getBridgeError(response) {
|
|
577
|
+
const hasError = !response || response.ok === false || (response.data && response.data.ok === false);
|
|
578
|
+
if (!hasError) return null;
|
|
579
|
+
return {
|
|
580
|
+
code: response?.error?.code || response?.data?.error?.code || 'BRIDGE_ERROR',
|
|
581
|
+
message: response?.error?.message || response?.data?.error?.message || 'Bridge request failed',
|
|
582
|
+
details: response?.error || response?.data?.error || null,
|
|
583
|
+
};
|
|
518
584
|
}
|
|
519
585
|
|
|
520
586
|
// ─── Output formatting ───────────────────────────────────────────────────────
|
|
@@ -625,96 +691,110 @@ export async function runImport(argv, deps = {}) {
|
|
|
625
691
|
}
|
|
626
692
|
}
|
|
627
693
|
|
|
628
|
-
const payload = {
|
|
629
|
-
items: items.map((item) => ({
|
|
630
|
-
title: item.title,
|
|
631
|
-
content: item._parseError ? '' : item.content,
|
|
632
|
-
source_path: item.source_path,
|
|
633
|
-
_parseError: item._parseError || null,
|
|
634
|
-
})),
|
|
635
|
-
folder: opts.folder || null,
|
|
636
|
-
tags: opts.tags || [],
|
|
637
|
-
conflict: opts.conflict || 'skip',
|
|
638
|
-
// Always check first - we'll prompt if needed
|
|
639
|
-
create_folder_if_missing: false,
|
|
640
|
-
dry_run: opts.dryRun,
|
|
641
|
-
};
|
|
642
|
-
|
|
643
694
|
const timeoutMs = Number(process.env.NOTESIA_CLI_BULK_TIMEOUT_MS) || 120_000;
|
|
644
695
|
const startMs = Date.now();
|
|
696
|
+
const itemBatches = splitItemsIntoPayloadBatches(items, opts);
|
|
697
|
+
const aggregate = {
|
|
698
|
+
created: 0,
|
|
699
|
+
skipped: 0,
|
|
700
|
+
overwritten: 0,
|
|
701
|
+
failed: 0,
|
|
702
|
+
items: [],
|
|
703
|
+
};
|
|
704
|
+
let createMissingFolder = false;
|
|
705
|
+
|
|
706
|
+
for (let batchIndex = 0; batchIndex < itemBatches.length; batchIndex += 1) {
|
|
707
|
+
const batchItems = itemBatches[batchIndex];
|
|
708
|
+
let payload = buildBulkImportPayload(batchItems, opts, createMissingFolder);
|
|
645
709
|
|
|
646
|
-
let response;
|
|
647
|
-
try {
|
|
648
710
|
if (opts.verbose) {
|
|
649
|
-
|
|
711
|
+
const estimatedBytes = estimateBridgeRequestBytes(payload);
|
|
712
|
+
process.stderr.write(
|
|
713
|
+
` [debug] calling bridge batch ${batchIndex + 1}/${itemBatches.length} `
|
|
714
|
+
+ `with ${batchItems.length} items (~${estimatedBytes} bytes), folder: "${opts.folder}"\n`
|
|
715
|
+
);
|
|
650
716
|
}
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
717
|
+
|
|
718
|
+
let response;
|
|
719
|
+
try {
|
|
720
|
+
response = await bridge({ op: 'bulk_import', payload }, { timeoutMs });
|
|
721
|
+
process.stderr.write(` [debug] bridge response ok: ${response?.ok}, hasData: ${!!response?.data}\n`);
|
|
722
|
+
if (opts.verbose || !response?.ok) {
|
|
723
|
+
process.stderr.write(` [debug] full response: ${JSON.stringify(response)}\n`);
|
|
724
|
+
}
|
|
725
|
+
} catch (err) {
|
|
726
|
+
const code = err.code || 'BRIDGE_ERROR';
|
|
727
|
+
if (code === 'NOTESIA_APP_NOT_RUNNING') {
|
|
728
|
+
process.stderr.write('NOTESIA_APP_NOT_RUNNING\n');
|
|
729
|
+
process.exit(1);
|
|
730
|
+
}
|
|
731
|
+
process.stderr.write(`[${code}] ${err.message}\n`);
|
|
660
732
|
process.exit(1);
|
|
661
733
|
}
|
|
662
|
-
process.stderr.write(`[${code}] ${err.message}\n`);
|
|
663
|
-
process.exit(1);
|
|
664
|
-
}
|
|
665
734
|
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
// Handle folder not found - prompt user to create (only if --create-folder is set)
|
|
673
|
-
const errorData = response?.error || response?.data?.error;
|
|
674
|
-
if (code === 'FOLDER_NOT_FOUND' && opts.folder && opts.createFolder) {
|
|
675
|
-
const folderName = errorData?.folderName || opts.folder;
|
|
735
|
+
let bridgeError = getBridgeError(response);
|
|
736
|
+
|
|
737
|
+
// Folder prompt flow happens once, then the same batch is retried with create_folder_if_missing=true.
|
|
738
|
+
if (bridgeError?.code === 'FOLDER_NOT_FOUND' && opts.folder && opts.createFolder && !createMissingFolder) {
|
|
739
|
+
const folderName = bridgeError?.details?.folderName || opts.folder;
|
|
676
740
|
process.stdout.write(`\nFolder "${folderName}" does not exist in Notesia.\n`);
|
|
677
741
|
const shouldCreate = await promptYesNo('Do you want to create it and continue import? [y/N] ');
|
|
678
|
-
if (shouldCreate) {
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
742
|
+
if (!shouldCreate) {
|
|
743
|
+
process.stderr.write('Import cancelled.\n');
|
|
744
|
+
process.exit(1);
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
createMissingFolder = true;
|
|
748
|
+
payload = buildBulkImportPayload(batchItems, opts, true);
|
|
749
|
+
try {
|
|
750
|
+
response = await bridge({ op: 'bulk_import', payload }, { timeoutMs });
|
|
751
|
+
process.stderr.write(` [debug] bridge response ok: ${response?.ok}, hasData: ${!!response?.data}\n`);
|
|
752
|
+
if (opts.verbose || !response?.ok) {
|
|
753
|
+
process.stderr.write(` [debug] full response: ${JSON.stringify(response)}\n`);
|
|
754
|
+
}
|
|
755
|
+
} catch (err) {
|
|
756
|
+
const code = err.code || 'BRIDGE_ERROR';
|
|
757
|
+
if (code === 'NOTESIA_APP_NOT_RUNNING') {
|
|
758
|
+
process.stderr.write('NOTESIA_APP_NOT_RUNNING\n');
|
|
688
759
|
process.exit(1);
|
|
689
760
|
}
|
|
690
|
-
|
|
691
|
-
} else {
|
|
692
|
-
process.stderr.write('Import cancelled.\n');
|
|
761
|
+
process.stderr.write(`[${code}] ${err.message}\n`);
|
|
693
762
|
process.exit(1);
|
|
694
763
|
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
764
|
+
|
|
765
|
+
bridgeError = getBridgeError(response);
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
if (bridgeError) {
|
|
769
|
+
if (bridgeError.code === 'FOLDER_NOT_FOUND' && opts.folder && !opts.createFolder) {
|
|
770
|
+
process.stderr.write(`[${bridgeError.code}] ${bridgeError.message}\n`);
|
|
771
|
+
process.exit(1);
|
|
772
|
+
}
|
|
773
|
+
process.stderr.write(`[${bridgeError.code}] ${bridgeError.message}\n`);
|
|
698
774
|
process.exit(1);
|
|
699
|
-
}
|
|
700
|
-
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
if (!response?.data) {
|
|
778
|
+
process.stderr.write(`[BRIDGE_ERROR] No data in bridge response. Raw response: ${JSON.stringify(response)}\n`);
|
|
701
779
|
process.exit(1);
|
|
702
780
|
}
|
|
703
|
-
}
|
|
704
781
|
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
782
|
+
aggregate.created += response.data.created ?? 0;
|
|
783
|
+
aggregate.skipped += response.data.skipped ?? 0;
|
|
784
|
+
aggregate.overwritten += response.data.overwritten ?? 0;
|
|
785
|
+
aggregate.failed += response.data.failed ?? 0;
|
|
786
|
+
if (Array.isArray(response.data.items)) {
|
|
787
|
+
aggregate.items.push(...response.data.items);
|
|
788
|
+
}
|
|
709
789
|
}
|
|
710
790
|
|
|
711
791
|
const result = {
|
|
712
|
-
created:
|
|
713
|
-
skipped:
|
|
714
|
-
overwritten:
|
|
715
|
-
failed:
|
|
716
|
-
elapsedMs:
|
|
717
|
-
items:
|
|
792
|
+
created: aggregate.created,
|
|
793
|
+
skipped: aggregate.skipped,
|
|
794
|
+
overwritten: aggregate.overwritten,
|
|
795
|
+
failed: aggregate.failed,
|
|
796
|
+
elapsedMs: Date.now() - startMs,
|
|
797
|
+
items: aggregate.items,
|
|
718
798
|
};
|
|
719
799
|
printResult(result, { format: opts.format, quiet: opts.quiet });
|
|
720
800
|
process.exit(result.failed === 0 ? 0 : 1);
|