@clarionstack/notesia 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/package.json +6 -2
- package/src/commands.mjs +1 -1
- package/src/import.mjs +467 -66
package/README.md
CHANGED
|
@@ -38,7 +38,7 @@ Recursively discovers files in a directory and imports them as notes into Notesi
|
|
|
38
38
|
| Option | Description | Default |
|
|
39
39
|
|---|---|---|
|
|
40
40
|
| `<path>` | Source directory or file (required) | — |
|
|
41
|
-
| `--ext <list>` | Comma-separated file extensions | `md,txt` |
|
|
41
|
+
| `--ext <list>` | Comma-separated file extensions | `md,txt,pdf,docx,json` |
|
|
42
42
|
| `--folder <name>` | Target folder name (created if absent) | — |
|
|
43
43
|
| `--tags <list>` | Comma-separated tags for all imported notes | — |
|
|
44
44
|
| `--conflict <mode>` | `skip` or `overwrite` existing notes by title | `skip` |
|
|
@@ -47,7 +47,9 @@ Recursively discovers files in a directory and imports them as notes into Notesi
|
|
|
47
47
|
| `--quiet` | Suppress stdout output | — |
|
|
48
48
|
| `--verbose` | Print per-file progress to stderr | — |
|
|
49
49
|
|
|
50
|
-
Supported file types: `.md`, `.txt`, `.html`, `.pdf
|
|
50
|
+
Supported file types: `.md`, `.txt`, `.html`, `.pdf`, `.docx`, `.json`.
|
|
51
|
+
|
|
52
|
+
JSON imports are transformed into flattened key-path text (with truncation guards for very large arrays/strings) to improve embedding quality.
|
|
51
53
|
|
|
52
54
|
Notes exceeding 100,000 characters are automatically split into multiple parts.
|
|
53
55
|
|
|
@@ -84,8 +86,8 @@ notesia import ~/Documents/notes
|
|
|
84
86
|
# Import into a specific folder with tags
|
|
85
87
|
notesia import ./project-docs --folder "Work" --tags "project,2026"
|
|
86
88
|
|
|
87
|
-
# Import HTML and
|
|
88
|
-
notesia import ./archive --ext md,txt,html,pdf
|
|
89
|
+
# Import HTML, PDF, DOCX and JSON files too
|
|
90
|
+
notesia import ./archive --ext md,txt,html,pdf,docx,json
|
|
89
91
|
|
|
90
92
|
# Overwrite existing notes with the same title
|
|
91
93
|
notesia import ./updated-notes --conflict overwrite
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@clarionstack/notesia",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.8",
|
|
5
5
|
"description": "Notesia desktop bridge CLI — read, write, and bulk-import notes",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"files": [
|
|
@@ -23,5 +23,9 @@
|
|
|
23
23
|
],
|
|
24
24
|
"engines": {
|
|
25
25
|
"node": ">=20"
|
|
26
|
+
},
|
|
27
|
+
"dependencies": {
|
|
28
|
+
"pdfjs-dist": "^4.0.379",
|
|
29
|
+
"jszip": "^3.10.1"
|
|
26
30
|
}
|
|
27
|
-
}
|
|
31
|
+
}
|
package/src/commands.mjs
CHANGED
|
@@ -311,7 +311,7 @@ function usageText() {
|
|
|
311
311
|
' auth status [--format text|json]',
|
|
312
312
|
'',
|
|
313
313
|
'Bulk Import:',
|
|
314
|
-
' import <path> [--ext md,txt] [--folder <name>] [--tags <list>]',
|
|
314
|
+
' import <path> [--ext md,txt,pdf,docx,json] [--folder <name>] [--tags <list>]',
|
|
315
315
|
' [--conflict skip|overwrite] [--dry-run]',
|
|
316
316
|
' [--format text|json] [--quiet] [--verbose]',
|
|
317
317
|
'',
|
package/src/import.mjs
CHANGED
|
@@ -6,13 +6,21 @@
|
|
|
6
6
|
* Dexie writes happen in the renderer via the bulk_import bridge op.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { createReadStream } from 'node:fs';
|
|
10
9
|
import fs from 'node:fs/promises';
|
|
11
10
|
import path from 'node:path';
|
|
12
11
|
import readline from 'node:readline';
|
|
13
12
|
import { requestBridge } from './client.mjs';
|
|
14
13
|
|
|
15
14
|
const NOTE_CHAR_LIMIT = 100_000;
|
|
15
|
+
const BRIDGE_MAX_REQUEST_BYTES = 1024 * 1024;
|
|
16
|
+
const BRIDGE_TARGET_REQUEST_BYTES = Math.floor(BRIDGE_MAX_REQUEST_BYTES * 0.9);
|
|
17
|
+
const DEFAULT_EXTENSIONS = ['md', 'txt', 'pdf', 'docx', 'json'];
|
|
18
|
+
const JSON_MAX_DEPTH = 12;
|
|
19
|
+
const JSON_MAX_ENTRIES = 5_000;
|
|
20
|
+
const JSON_MAX_ARRAY_ITEMS = 100;
|
|
21
|
+
const JSON_MAX_STRING_LENGTH = 500;
|
|
22
|
+
const JSON_MAX_TOP_LEVEL_KEYS = 25;
|
|
23
|
+
const SIMPLE_JSON_KEY_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
16
24
|
|
|
17
25
|
const USAGE = [
|
|
18
26
|
'Usage: notesia import <path> [options]',
|
|
@@ -22,7 +30,7 @@ const USAGE = [
|
|
|
22
30
|
'',
|
|
23
31
|
'Options:',
|
|
24
32
|
' --path <dir> Path to directory to import',
|
|
25
|
-
|
|
33
|
+
` --ext <list> Comma-separated extensions to import (default: ${DEFAULT_EXTENSIONS.join(',')})`,
|
|
26
34
|
' --folder <name> Target folder name in Notesia',
|
|
27
35
|
' --tags <list> Comma-separated tags to apply to all imported notes',
|
|
28
36
|
' --conflict <mode> Conflict resolution: skip | overwrite (default: skip)',
|
|
@@ -47,7 +55,7 @@ export function parseImportArgs(argv) {
|
|
|
47
55
|
|
|
48
56
|
const opts = {
|
|
49
57
|
importPath: null,
|
|
50
|
-
ext: [
|
|
58
|
+
ext: [...DEFAULT_EXTENSIONS],
|
|
51
59
|
folder: null,
|
|
52
60
|
tags: [],
|
|
53
61
|
conflict: 'skip',
|
|
@@ -76,6 +84,10 @@ export function parseImportArgs(argv) {
|
|
|
76
84
|
} else if (token === '--ext') {
|
|
77
85
|
const val = args[++i];
|
|
78
86
|
if (!val || val.startsWith('--')) { process.stderr.write('Error: --ext requires a value\n\n' + USAGE); process.exit(2); }
|
|
87
|
+
opts.ext = val.split(',').map((e) => e.trim().toLowerCase()).filter(Boolean);
|
|
88
|
+
} else if (token === '--folder') {
|
|
89
|
+
const val = args[++i];
|
|
90
|
+
if (!val || val.startsWith('--')) { process.stderr.write('Error: --folder requires a value\n\n' + USAGE); process.exit(2); }
|
|
79
91
|
opts.folder = val;
|
|
80
92
|
} else if (token === '--tags') {
|
|
81
93
|
const val = args[++i];
|
|
@@ -149,9 +161,19 @@ async function discoverFiles(rootPath, extensions) {
|
|
|
149
161
|
|
|
150
162
|
// ─── Content parsing ─────────────────────────────────────────────────────────
|
|
151
163
|
|
|
152
|
-
function titleFromPath(filePath) {
|
|
153
|
-
|
|
154
|
-
|
|
164
|
+
function titleFromPath(filePath, relativePath) {
|
|
165
|
+
// Use relative path to include folder structure in title
|
|
166
|
+
// e.g., "folder1/SKILL.md" → "folder1/SKILL"
|
|
167
|
+
const baseName = path.basename(relativePath, path.extname(relativePath));
|
|
168
|
+
const folderName = path.dirname(relativePath);
|
|
169
|
+
const stem = baseName.replace(/[-_]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
170
|
+
|
|
171
|
+
// If file is in a subfolder, prepend folder to title
|
|
172
|
+
if (folderName && folderName !== '.') {
|
|
173
|
+
const folderStem = folderName.replace(/[-_]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
174
|
+
return `${folderStem}/${stem}`;
|
|
175
|
+
}
|
|
176
|
+
return stem;
|
|
155
177
|
}
|
|
156
178
|
|
|
157
179
|
function stripHtml(html) {
|
|
@@ -163,8 +185,185 @@ function stripHtml(html) {
|
|
|
163
185
|
.trim();
|
|
164
186
|
}
|
|
165
187
|
|
|
166
|
-
|
|
167
|
-
|
|
188
|
+
function decodeXmlEntities(value) {
|
|
189
|
+
if (!value) return '';
|
|
190
|
+
return value
|
|
191
|
+
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
|
192
|
+
.replace(/&/g, '&')
|
|
193
|
+
.replace(/</g, '<')
|
|
194
|
+
.replace(/>/g, '>')
|
|
195
|
+
.replace(/"/g, '"')
|
|
196
|
+
.replace(/'/g, "'")
|
|
197
|
+
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
|
|
198
|
+
.replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10)))
|
|
199
|
+
.replace(/\u00A0/g, ' ');
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function sharedLeadingWhitespace(a, b) {
|
|
203
|
+
const max = Math.min(a.length, b.length);
|
|
204
|
+
let i = 0;
|
|
205
|
+
while (i < max && a[i] === b[i]) i++;
|
|
206
|
+
return a.slice(0, i);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function normalizeFencedCodeBlocks(markdown) {
|
|
210
|
+
if (!markdown || !markdown.includes('```')) return markdown;
|
|
211
|
+
|
|
212
|
+
return markdown.replace(/```([^\n]*)\n([\s\S]*?)\n```/g, (full, lang, body) => {
|
|
213
|
+
const lines = String(body).replace(/\u00A0/g, ' ').split('\n');
|
|
214
|
+
const nonEmpty = lines.filter((line) => line.trim().length > 0);
|
|
215
|
+
if (nonEmpty.length === 0) return full;
|
|
216
|
+
|
|
217
|
+
let commonIndent = null;
|
|
218
|
+
for (const line of nonEmpty) {
|
|
219
|
+
const indent = line.match(/^[ \t]*/)?.[0] || '';
|
|
220
|
+
if (!indent) {
|
|
221
|
+
commonIndent = '';
|
|
222
|
+
break;
|
|
223
|
+
}
|
|
224
|
+
commonIndent = commonIndent === null ? indent : sharedLeadingWhitespace(commonIndent, indent);
|
|
225
|
+
if (!commonIndent) break;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
if (!commonIndent) {
|
|
229
|
+
return `\`\`\`${lang}\n${lines.join('\n')}\n\`\`\``;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const dedented = lines.map((line) =>
|
|
233
|
+
line.startsWith(commonIndent) ? line.slice(commonIndent.length) : line
|
|
234
|
+
);
|
|
235
|
+
return `\`\`\`${lang}\n${dedented.join('\n')}\n\`\`\``;
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function appendJsonPath(basePath, key) {
|
|
240
|
+
if (SIMPLE_JSON_KEY_RE.test(key)) {
|
|
241
|
+
return basePath === '$' ? `$.${key}` : `${basePath}.${key}`;
|
|
242
|
+
}
|
|
243
|
+
const escapedKey = String(key).replaceAll('\\', '\\\\').replaceAll('"', '\\"');
|
|
244
|
+
return `${basePath}["${escapedKey}"]`;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function formatJsonValue(value) {
|
|
248
|
+
if (typeof value === 'string') {
|
|
249
|
+
const normalized = value.replace(/\s+/g, ' ').trim();
|
|
250
|
+
if (normalized.length > JSON_MAX_STRING_LENGTH) {
|
|
251
|
+
const truncatedChars = normalized.length - JSON_MAX_STRING_LENGTH;
|
|
252
|
+
const shortened = normalized.slice(0, JSON_MAX_STRING_LENGTH);
|
|
253
|
+
return `${JSON.stringify(shortened)} [truncated ${truncatedChars} chars]`;
|
|
254
|
+
}
|
|
255
|
+
return JSON.stringify(normalized);
|
|
256
|
+
}
|
|
257
|
+
if (value === null) return 'null';
|
|
258
|
+
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
|
259
|
+
return JSON.stringify(value);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function flattenJsonForEmbedding(root) {
|
|
263
|
+
const lines = [];
|
|
264
|
+
const notes = [];
|
|
265
|
+
const stack = [{ path: '$', value: root, depth: 0 }];
|
|
266
|
+
let hitEntryLimit = false;
|
|
267
|
+
|
|
268
|
+
while (stack.length > 0) {
|
|
269
|
+
if (lines.length >= JSON_MAX_ENTRIES) {
|
|
270
|
+
hitEntryLimit = true;
|
|
271
|
+
break;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const current = stack.pop();
|
|
275
|
+
const { path: currentPath, value, depth } = current;
|
|
276
|
+
|
|
277
|
+
if (value === null || typeof value !== 'object') {
|
|
278
|
+
lines.push(`${currentPath}: ${formatJsonValue(value)}`);
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
if (depth >= JSON_MAX_DEPTH) {
|
|
283
|
+
lines.push(`${currentPath}: [max depth ${JSON_MAX_DEPTH} reached]`);
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
if (Array.isArray(value)) {
|
|
288
|
+
lines.push(`${currentPath}: [array length=${value.length}]`);
|
|
289
|
+
const keptItems = Math.min(value.length, JSON_MAX_ARRAY_ITEMS);
|
|
290
|
+
|
|
291
|
+
for (let i = keptItems - 1; i >= 0; i -= 1) {
|
|
292
|
+
stack.push({
|
|
293
|
+
path: `${currentPath}[${i}]`,
|
|
294
|
+
value: value[i],
|
|
295
|
+
depth: depth + 1,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
if (value.length > keptItems) {
|
|
300
|
+
lines.push(`${currentPath}: [omitted after item ${keptItems - 1}] (${value.length - keptItems} items omitted)`);
|
|
301
|
+
}
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const entries = Object.entries(value);
|
|
306
|
+
if (entries.length === 0) {
|
|
307
|
+
lines.push(`${currentPath}: {}`);
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
for (let i = entries.length - 1; i >= 0; i -= 1) {
|
|
312
|
+
const [key, childValue] = entries[i];
|
|
313
|
+
stack.push({
|
|
314
|
+
path: appendJsonPath(currentPath, key),
|
|
315
|
+
value: childValue,
|
|
316
|
+
depth: depth + 1,
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
if (hitEntryLimit) {
|
|
322
|
+
notes.push(`[truncated] Flattened field limit reached (${JSON_MAX_ENTRIES}); remaining fields omitted.`);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
return { lines, notes };
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function formatJsonForEmbedding(jsonValue, sourcePath = '') {
|
|
329
|
+
const rootType = Array.isArray(jsonValue)
|
|
330
|
+
? 'array'
|
|
331
|
+
: (jsonValue === null ? 'null' : typeof jsonValue);
|
|
332
|
+
const topLevelKeys = rootType === 'object' && jsonValue
|
|
333
|
+
? Object.keys(jsonValue)
|
|
334
|
+
: [];
|
|
335
|
+
const { lines, notes } = flattenJsonForEmbedding(jsonValue);
|
|
336
|
+
|
|
337
|
+
const output = [
|
|
338
|
+
'# JSON Import',
|
|
339
|
+
'',
|
|
340
|
+
`Source: ${sourcePath || 'unknown'}`,
|
|
341
|
+
`Root type: ${rootType}`,
|
|
342
|
+
];
|
|
343
|
+
|
|
344
|
+
if (rootType === 'array') {
|
|
345
|
+
output.push(`Top-level items: ${jsonValue.length}`);
|
|
346
|
+
} else if (rootType === 'object') {
|
|
347
|
+
const visibleKeys = topLevelKeys.slice(0, JSON_MAX_TOP_LEVEL_KEYS);
|
|
348
|
+
const suffix = topLevelKeys.length > visibleKeys.length
|
|
349
|
+
? ` (+${topLevelKeys.length - visibleKeys.length} more)`
|
|
350
|
+
: '';
|
|
351
|
+
output.push(`Top-level keys (${topLevelKeys.length}): ${visibleKeys.join(', ')}${suffix}`);
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
output.push('', '## Flattened Fields', '');
|
|
355
|
+
output.push(...(lines.length > 0 ? lines : ['$: [empty document]']));
|
|
356
|
+
|
|
357
|
+
if (notes.length > 0) {
|
|
358
|
+
output.push('', '## Import Notes', '');
|
|
359
|
+
output.push(...notes.map((note) => `- ${note}`));
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
return output.join('\n').trim();
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
async function parseFile(absolutePath, extension, relativePath) {
|
|
366
|
+
const defaultTitle = titleFromPath(absolutePath, relativePath);
|
|
168
367
|
try {
|
|
169
368
|
if (extension === 'md' || extension === 'txt') {
|
|
170
369
|
const body = await fs.readFile(absolutePath, 'utf8');
|
|
@@ -176,23 +375,114 @@ async function parseFile(absolutePath, extension) {
|
|
|
176
375
|
const htmlTitle = titleMatch ? titleMatch[1].replace(/\s+/g, ' ').trim() : null;
|
|
177
376
|
return { title: htmlTitle || defaultTitle, body: stripHtml(raw) };
|
|
178
377
|
}
|
|
378
|
+
if (extension === 'json') {
|
|
379
|
+
const raw = await fs.readFile(absolutePath, 'utf8');
|
|
380
|
+
let parsedJson;
|
|
381
|
+
try {
|
|
382
|
+
parsedJson = JSON.parse(raw.replace(/^\uFEFF/, ''));
|
|
383
|
+
} catch (err) {
|
|
384
|
+
return { title: defaultTitle, body: '', error: `Invalid JSON: ${err.message}` };
|
|
385
|
+
}
|
|
386
|
+
return {
|
|
387
|
+
title: defaultTitle,
|
|
388
|
+
body: formatJsonForEmbedding(parsedJson, relativePath || path.basename(absolutePath)),
|
|
389
|
+
};
|
|
390
|
+
}
|
|
179
391
|
if (extension === 'pdf') {
|
|
180
|
-
//
|
|
392
|
+
// Use pdfjs-dist legacy build for Node.js PDF text extraction
|
|
181
393
|
try {
|
|
182
|
-
const
|
|
183
|
-
pdfjsLib.GlobalWorkerOptions.workerSrc = '';
|
|
394
|
+
const pdfjsLib = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
|
184
395
|
const data = await fs.readFile(absolutePath);
|
|
185
|
-
const pdf = await pdfjsLib.getDocument({ data: new Uint8Array(data)
|
|
396
|
+
const pdf = await pdfjsLib.getDocument({ data: new Uint8Array(data.buffer) }).promise;
|
|
186
397
|
const pages = [];
|
|
187
398
|
for (let i = 1; i <= pdf.numPages; i++) {
|
|
188
399
|
const page = await pdf.getPage(i);
|
|
189
400
|
const content = await page.getTextContent();
|
|
190
|
-
|
|
401
|
+
|
|
402
|
+
// Extract text with better structure preservation
|
|
403
|
+
const items = content.items;
|
|
404
|
+
let pageText = '';
|
|
405
|
+
let lastY = null;
|
|
406
|
+
|
|
407
|
+
for (const item of items) {
|
|
408
|
+
if (lastY !== null && Math.abs(item.transform[5] - lastY) > 5) {
|
|
409
|
+
// Different line
|
|
410
|
+
pageText += '\n';
|
|
411
|
+
} else if (pageText !== '') {
|
|
412
|
+
// Same line - add space between items
|
|
413
|
+
pageText += ' ';
|
|
414
|
+
}
|
|
415
|
+
pageText += item.str;
|
|
416
|
+
lastY = item.transform[5];
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
pages.push(pageText.trim());
|
|
191
420
|
page.cleanup();
|
|
192
421
|
}
|
|
422
|
+
|
|
193
423
|
let title = defaultTitle;
|
|
194
424
|
try { const meta = await pdf.getMetadata(); if (meta?.info?.Title?.trim()) title = meta.info.Title.trim(); } catch {}
|
|
195
|
-
|
|
425
|
+
|
|
426
|
+
// Join pages with double newline and clean up
|
|
427
|
+
const body = pages.join('\n\n')
|
|
428
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
429
|
+
.replace(/[ \t]+/g, ' ')
|
|
430
|
+
.trim();
|
|
431
|
+
|
|
432
|
+
return { title, body };
|
|
433
|
+
} catch (err) {
|
|
434
|
+
return { title: defaultTitle, body: '', error: err.message };
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
if (extension === 'docx') {
|
|
438
|
+
// Use JSZip for DOCX text extraction
|
|
439
|
+
try {
|
|
440
|
+
const JSZip = (await import('jszip')).default;
|
|
441
|
+
const data = await fs.readFile(absolutePath);
|
|
442
|
+
const zip = await JSZip.loadAsync(data);
|
|
443
|
+
const docXml = await zip.file('word/document.xml')?.async('string');
|
|
444
|
+
if (!docXml) {
|
|
445
|
+
return { title: defaultTitle, body: '', error: 'Invalid DOCX file: missing document.xml' };
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Extract title from docProps/core.xml if available
|
|
449
|
+
let title = defaultTitle;
|
|
450
|
+
try {
|
|
451
|
+
const coreXml = await zip.file('docProps/core.xml')?.async('string');
|
|
452
|
+
if (coreXml) {
|
|
453
|
+
const titleMatch = coreXml.match(/<dc:title[^>]*>([^<]*)<\/dc:title>/);
|
|
454
|
+
if (titleMatch && titleMatch[1]?.trim()) {
|
|
455
|
+
title = titleMatch[1].trim();
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
} catch {}
|
|
459
|
+
|
|
460
|
+
// Parse XML to extract plain text with proper line breaks
|
|
461
|
+
let text = docXml;
|
|
462
|
+
|
|
463
|
+
// Preserve CDATA text content before stripping tags.
|
|
464
|
+
text = text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1');
|
|
465
|
+
|
|
466
|
+
// Handle paragraph spacing - use double newline for paragraphs
|
|
467
|
+
text = text
|
|
468
|
+
.replace(/<\/w:p>/g, '\n\n')
|
|
469
|
+
.replace(/<w:tab[^>]*\/>/g, '\t')
|
|
470
|
+
.replace(/<w:br[^>]*\/>/g, '\n')
|
|
471
|
+
.replace(/<w:cr[^>]*\/>/g, '\n');
|
|
472
|
+
|
|
473
|
+
// Remove all remaining XML tags
|
|
474
|
+
text = text
|
|
475
|
+
.replace(/<[^>]+>/g, '')
|
|
476
|
+
.replace(/\r\n/g, '\n')
|
|
477
|
+
.replace(/\r/g, '\n');
|
|
478
|
+
|
|
479
|
+
text = decodeXmlEntities(text)
|
|
480
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
481
|
+
.trim();
|
|
482
|
+
|
|
483
|
+
text = normalizeFencedCodeBlocks(text);
|
|
484
|
+
|
|
485
|
+
return { title, body: text };
|
|
196
486
|
} catch (err) {
|
|
197
487
|
return { title: defaultTitle, body: '', error: err.message };
|
|
198
488
|
}
|
|
@@ -214,12 +504,83 @@ function splitContent(title, body) {
|
|
|
214
504
|
if (remaining.length <= NOTE_CHAR_LIMIT) { chunks.push(remaining); break; }
|
|
215
505
|
const searchStart = Math.max(0, NOTE_CHAR_LIMIT - 500);
|
|
216
506
|
const slice = remaining.slice(searchStart, NOTE_CHAR_LIMIT);
|
|
217
|
-
const
|
|
218
|
-
|
|
507
|
+
const lastParagraphBreak = slice.lastIndexOf('\n\n');
|
|
508
|
+
if (lastParagraphBreak !== -1) {
|
|
509
|
+
chunks.push(remaining.slice(0, searchStart + lastParagraphBreak + 2));
|
|
510
|
+
remaining = remaining.slice(searchStart + lastParagraphBreak + 2);
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
const lastLineBreak = slice.lastIndexOf('\n');
|
|
515
|
+
const splitAt = lastLineBreak !== -1 ? searchStart + lastLineBreak + 1 : NOTE_CHAR_LIMIT;
|
|
219
516
|
chunks.push(remaining.slice(0, splitAt));
|
|
220
517
|
remaining = remaining.slice(splitAt);
|
|
221
518
|
}
|
|
222
|
-
return chunks.map((content, i) => ({
|
|
519
|
+
return chunks.map((content, i) => ({
|
|
520
|
+
title: i === 0 ? title : `${title}_${i + 1}`,
|
|
521
|
+
content,
|
|
522
|
+
}));
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function normalizeImportItem(item) {
|
|
526
|
+
return {
|
|
527
|
+
title: item.title,
|
|
528
|
+
content: item._parseError ? '' : item.content,
|
|
529
|
+
source_path: item.source_path,
|
|
530
|
+
_parseError: item._parseError || null,
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function buildBulkImportPayload(items, opts, createFolderIfMissing = false) {
|
|
535
|
+
return {
|
|
536
|
+
items: items.map(normalizeImportItem),
|
|
537
|
+
folder: opts.folder || null,
|
|
538
|
+
tags: opts.tags || [],
|
|
539
|
+
conflict: opts.conflict || 'skip',
|
|
540
|
+
create_folder_if_missing: createFolderIfMissing,
|
|
541
|
+
dry_run: opts.dryRun,
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function estimateBridgeRequestBytes(payload) {
|
|
546
|
+
return Buffer.byteLength(JSON.stringify({ op: 'bulk_import', payload }), 'utf8');
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
function splitItemsIntoPayloadBatches(items, opts) {
|
|
550
|
+
if (items.length === 0) return [[]];
|
|
551
|
+
|
|
552
|
+
const batches = [];
|
|
553
|
+
let current = [];
|
|
554
|
+
|
|
555
|
+
for (const item of items) {
|
|
556
|
+
const candidate = [...current, item];
|
|
557
|
+
const candidatePayload = buildBulkImportPayload(candidate, opts, false);
|
|
558
|
+
const candidateBytes = estimateBridgeRequestBytes(candidatePayload);
|
|
559
|
+
|
|
560
|
+
if (current.length > 0 && candidateBytes > BRIDGE_TARGET_REQUEST_BYTES) {
|
|
561
|
+
batches.push(current);
|
|
562
|
+
current = [item];
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
current = candidate;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
if (current.length > 0) {
|
|
570
|
+
batches.push(current);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
return batches;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
function getBridgeError(response) {
|
|
577
|
+
const hasError = !response || response.ok === false || (response.data && response.data.ok === false);
|
|
578
|
+
if (!hasError) return null;
|
|
579
|
+
return {
|
|
580
|
+
code: response?.error?.code || response?.data?.error?.code || 'BRIDGE_ERROR',
|
|
581
|
+
message: response?.error?.message || response?.data?.error?.message || 'Bridge request failed',
|
|
582
|
+
details: response?.error || response?.data?.error || null,
|
|
583
|
+
};
|
|
223
584
|
}
|
|
224
585
|
|
|
225
586
|
// ─── Output formatting ───────────────────────────────────────────────────────
|
|
@@ -252,7 +613,7 @@ function printResult(result, opts) {
|
|
|
252
613
|
if (item.status === 'created') {
|
|
253
614
|
reason = `noteId: ${item.noteId}`;
|
|
254
615
|
} else if (item.status === 'skipped') {
|
|
255
|
-
reason = `noteId: ${item.noteId}`;
|
|
616
|
+
reason = `Note with same title already exists in target folder (noteId: ${item.noteId})`;
|
|
256
617
|
} else if (item.status === 'overwritten') {
|
|
257
618
|
reason = `noteId: ${item.noteId}`;
|
|
258
619
|
} else if (item.status === 'failed') {
|
|
@@ -311,7 +672,7 @@ export async function runImport(argv, deps = {}) {
|
|
|
311
672
|
for (const file of files) {
|
|
312
673
|
if (opts.verbose) process.stderr.write(` [parsing] ${file.relativePath}\n`);
|
|
313
674
|
|
|
314
|
-
const parsed = await parseFile(file.absolutePath, file.extension);
|
|
675
|
+
const parsed = await parseFile(file.absolutePath, file.extension, file.relativePath);
|
|
315
676
|
|
|
316
677
|
if (parsed.error && !parsed.body) {
|
|
317
678
|
if (opts.verbose) process.stderr.write(` [failed] ${file.relativePath} — ${parsed.error}\n`);
|
|
@@ -330,71 +691,111 @@ export async function runImport(argv, deps = {}) {
|
|
|
330
691
|
}
|
|
331
692
|
}
|
|
332
693
|
|
|
333
|
-
const payload = {
|
|
334
|
-
items: items.map((item) => ({
|
|
335
|
-
title: item.title,
|
|
336
|
-
content: item._parseError ? '' : item.content,
|
|
337
|
-
source_path: item.source_path,
|
|
338
|
-
_parseError: item._parseError || null,
|
|
339
|
-
})),
|
|
340
|
-
folder: opts.folder,
|
|
341
|
-
tags: opts.tags,
|
|
342
|
-
conflict: opts.conflict,
|
|
343
|
-
// Always check first - we'll prompt if needed
|
|
344
|
-
create_folder_if_missing: false,
|
|
345
|
-
dry_run: opts.dryRun,
|
|
346
|
-
};
|
|
347
|
-
|
|
348
694
|
const timeoutMs = Number(process.env.NOTESIA_CLI_BULK_TIMEOUT_MS) || 120_000;
|
|
349
695
|
const startMs = Date.now();
|
|
696
|
+
const itemBatches = splitItemsIntoPayloadBatches(items, opts);
|
|
697
|
+
const aggregate = {
|
|
698
|
+
created: 0,
|
|
699
|
+
skipped: 0,
|
|
700
|
+
overwritten: 0,
|
|
701
|
+
failed: 0,
|
|
702
|
+
items: [],
|
|
703
|
+
};
|
|
704
|
+
let createMissingFolder = false;
|
|
350
705
|
|
|
351
|
-
let
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
process.stderr.write(
|
|
706
|
+
for (let batchIndex = 0; batchIndex < itemBatches.length; batchIndex += 1) {
|
|
707
|
+
const batchItems = itemBatches[batchIndex];
|
|
708
|
+
let payload = buildBulkImportPayload(batchItems, opts, createMissingFolder);
|
|
709
|
+
|
|
710
|
+
if (opts.verbose) {
|
|
711
|
+
const estimatedBytes = estimateBridgeRequestBytes(payload);
|
|
712
|
+
process.stderr.write(
|
|
713
|
+
` [debug] calling bridge batch ${batchIndex + 1}/${itemBatches.length} `
|
|
714
|
+
+ `with ${batchItems.length} items (~${estimatedBytes} bytes), folder: "${opts.folder}"\n`
|
|
715
|
+
);
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
let response;
|
|
719
|
+
try {
|
|
720
|
+
response = await bridge({ op: 'bulk_import', payload }, { timeoutMs });
|
|
721
|
+
process.stderr.write(` [debug] bridge response ok: ${response?.ok}, hasData: ${!!response?.data}\n`);
|
|
722
|
+
if (opts.verbose || !response?.ok) {
|
|
723
|
+
process.stderr.write(` [debug] full response: ${JSON.stringify(response)}\n`);
|
|
724
|
+
}
|
|
725
|
+
} catch (err) {
|
|
726
|
+
const code = err.code || 'BRIDGE_ERROR';
|
|
727
|
+
if (code === 'NOTESIA_APP_NOT_RUNNING') {
|
|
728
|
+
process.stderr.write('NOTESIA_APP_NOT_RUNNING\n');
|
|
729
|
+
process.exit(1);
|
|
730
|
+
}
|
|
731
|
+
process.stderr.write(`[${code}] ${err.message}\n`);
|
|
358
732
|
process.exit(1);
|
|
359
733
|
}
|
|
360
|
-
process.stderr.write(`[${code}] ${err.message}\n`);
|
|
361
|
-
process.exit(1);
|
|
362
|
-
}
|
|
363
734
|
|
|
364
|
-
|
|
365
|
-
const code = response?.error?.code || 'BRIDGE_ERROR';
|
|
366
|
-
const message = response?.error?.message || 'Bridge request failed';
|
|
735
|
+
let bridgeError = getBridgeError(response);
|
|
367
736
|
|
|
368
|
-
//
|
|
369
|
-
if (code === 'FOLDER_NOT_FOUND' && opts.folder && opts.createFolder) {
|
|
370
|
-
const folderName =
|
|
737
|
+
// Folder prompt flow happens once, then the same batch is retried with create_folder_if_missing=true.
|
|
738
|
+
if (bridgeError?.code === 'FOLDER_NOT_FOUND' && opts.folder && opts.createFolder && !createMissingFolder) {
|
|
739
|
+
const folderName = bridgeError?.details?.folderName || opts.folder;
|
|
371
740
|
process.stdout.write(`\nFolder "${folderName}" does not exist in Notesia.\n`);
|
|
372
741
|
const shouldCreate = await promptYesNo('Do you want to create it and continue import? [y/N] ');
|
|
373
|
-
if (shouldCreate) {
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
742
|
+
if (!shouldCreate) {
|
|
743
|
+
process.stderr.write('Import cancelled.\n');
|
|
744
|
+
process.exit(1);
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
createMissingFolder = true;
|
|
748
|
+
payload = buildBulkImportPayload(batchItems, opts, true);
|
|
749
|
+
try {
|
|
750
|
+
response = await bridge({ op: 'bulk_import', payload }, { timeoutMs });
|
|
751
|
+
process.stderr.write(` [debug] bridge response ok: ${response?.ok}, hasData: ${!!response?.data}\n`);
|
|
752
|
+
if (opts.verbose || !response?.ok) {
|
|
753
|
+
process.stderr.write(` [debug] full response: ${JSON.stringify(response)}\n`);
|
|
754
|
+
}
|
|
755
|
+
} catch (err) {
|
|
756
|
+
const code = err.code || 'BRIDGE_ERROR';
|
|
757
|
+
if (code === 'NOTESIA_APP_NOT_RUNNING') {
|
|
758
|
+
process.stderr.write('NOTESIA_APP_NOT_RUNNING\n');
|
|
381
759
|
process.exit(1);
|
|
382
760
|
}
|
|
383
|
-
|
|
384
|
-
process.
|
|
761
|
+
process.stderr.write(`[${code}] ${err.message}\n`);
|
|
762
|
+
process.exit(1);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
bridgeError = getBridgeError(response);
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
if (bridgeError) {
|
|
769
|
+
if (bridgeError.code === 'FOLDER_NOT_FOUND' && opts.folder && !opts.createFolder) {
|
|
770
|
+
process.stderr.write(`[${bridgeError.code}] ${bridgeError.message}\n`);
|
|
385
771
|
process.exit(1);
|
|
386
772
|
}
|
|
387
|
-
|
|
388
|
-
// User specified --no-create-folder, so just show the error
|
|
389
|
-
process.stderr.write(`[${code}] ${message}\n`);
|
|
773
|
+
process.stderr.write(`[${bridgeError.code}] ${bridgeError.message}\n`);
|
|
390
774
|
process.exit(1);
|
|
391
|
-
}
|
|
392
|
-
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
if (!response?.data) {
|
|
778
|
+
process.stderr.write(`[BRIDGE_ERROR] No data in bridge response. Raw response: ${JSON.stringify(response)}\n`);
|
|
393
779
|
process.exit(1);
|
|
394
780
|
}
|
|
781
|
+
|
|
782
|
+
aggregate.created += response.data.created ?? 0;
|
|
783
|
+
aggregate.skipped += response.data.skipped ?? 0;
|
|
784
|
+
aggregate.overwritten += response.data.overwritten ?? 0;
|
|
785
|
+
aggregate.failed += response.data.failed ?? 0;
|
|
786
|
+
if (Array.isArray(response.data.items)) {
|
|
787
|
+
aggregate.items.push(...response.data.items);
|
|
788
|
+
}
|
|
395
789
|
}
|
|
396
790
|
|
|
397
|
-
const result = {
|
|
791
|
+
const result = {
|
|
792
|
+
created: aggregate.created,
|
|
793
|
+
skipped: aggregate.skipped,
|
|
794
|
+
overwritten: aggregate.overwritten,
|
|
795
|
+
failed: aggregate.failed,
|
|
796
|
+
elapsedMs: Date.now() - startMs,
|
|
797
|
+
items: aggregate.items,
|
|
798
|
+
};
|
|
398
799
|
printResult(result, { format: opts.format, quiet: opts.quiet });
|
|
399
800
|
process.exit(result.failed === 0 ? 0 : 1);
|
|
400
801
|
}
|