@clarionstack/notesia 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/package.json +6 -2
- package/src/commands.mjs +1 -1
- package/src/import.mjs +452 -34
- package/src/io.mjs +3 -1
package/README.md
CHANGED
|
@@ -38,7 +38,7 @@ Recursively discovers files in a directory and imports them as notes into Notesi
|
|
|
38
38
|
| Option | Description | Default |
|
|
39
39
|
|---|---|---|
|
|
40
40
|
| `<path>` | Source directory or file (required) | — |
|
|
41
|
-
| `--ext <list>` | Comma-separated file extensions | `md,txt` |
|
|
41
|
+
| `--ext <list>` | Comma-separated file extensions | `md,txt,pdf,docx,json` |
|
|
42
42
|
| `--folder <name>` | Target folder name (created if absent) | — |
|
|
43
43
|
| `--tags <list>` | Comma-separated tags for all imported notes | — |
|
|
44
44
|
| `--conflict <mode>` | `skip` or `overwrite` existing notes by title | `skip` |
|
|
@@ -47,7 +47,9 @@ Recursively discovers files in a directory and imports them as notes into Notesi
|
|
|
47
47
|
| `--quiet` | Suppress stdout output | — |
|
|
48
48
|
| `--verbose` | Print per-file progress to stderr | — |
|
|
49
49
|
|
|
50
|
-
Supported file types: `.md`, `.txt`, `.html`, `.pdf
|
|
50
|
+
Supported file types: `.md`, `.txt`, `.html`, `.pdf`, `.docx`, `.json`.
|
|
51
|
+
|
|
52
|
+
JSON imports are transformed into flattened key-path text (with truncation guards for very large arrays/strings) to improve embedding quality.
|
|
51
53
|
|
|
52
54
|
Notes exceeding 100,000 characters are automatically split into multiple parts.
|
|
53
55
|
|
|
@@ -84,8 +86,8 @@ notesia import ~/Documents/notes
|
|
|
84
86
|
# Import into a specific folder with tags
|
|
85
87
|
notesia import ./project-docs --folder "Work" --tags "project,2026"
|
|
86
88
|
|
|
87
|
-
# Import HTML and
|
|
88
|
-
notesia import ./archive --ext md,txt,html,pdf
|
|
89
|
+
# Import HTML, PDF, DOCX and JSON files too
|
|
90
|
+
notesia import ./archive --ext md,txt,html,pdf,docx,json
|
|
89
91
|
|
|
90
92
|
# Overwrite existing notes with the same title
|
|
91
93
|
notesia import ./updated-notes --conflict overwrite
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@clarionstack/notesia",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.7",
|
|
5
5
|
"description": "Notesia desktop bridge CLI — read, write, and bulk-import notes",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"files": [
|
|
@@ -23,5 +23,9 @@
|
|
|
23
23
|
],
|
|
24
24
|
"engines": {
|
|
25
25
|
"node": ">=20"
|
|
26
|
+
},
|
|
27
|
+
"dependencies": {
|
|
28
|
+
"pdfjs-dist": "^4.0.379",
|
|
29
|
+
"jszip": "^3.10.1"
|
|
26
30
|
}
|
|
27
|
-
}
|
|
31
|
+
}
|
package/src/commands.mjs
CHANGED
|
@@ -311,7 +311,7 @@ function usageText() {
|
|
|
311
311
|
' auth status [--format text|json]',
|
|
312
312
|
'',
|
|
313
313
|
'Bulk Import:',
|
|
314
|
-
' import <path> [--ext md,txt] [--folder <name>] [--tags <list>]',
|
|
314
|
+
' import <path> [--ext md,txt,pdf,docx,json] [--folder <name>] [--tags <list>]',
|
|
315
315
|
' [--conflict skip|overwrite] [--dry-run]',
|
|
316
316
|
' [--format text|json] [--quiet] [--verbose]',
|
|
317
317
|
'',
|
package/src/import.mjs
CHANGED
|
@@ -6,21 +6,34 @@
|
|
|
6
6
|
* Dexie writes happen in the renderer via the bulk_import bridge op.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { createReadStream } from 'node:fs';
|
|
10
9
|
import fs from 'node:fs/promises';
|
|
11
10
|
import path from 'node:path';
|
|
11
|
+
import readline from 'node:readline';
|
|
12
12
|
import { requestBridge } from './client.mjs';
|
|
13
13
|
|
|
14
14
|
const NOTE_CHAR_LIMIT = 100_000;
|
|
15
|
+
const DEFAULT_EXTENSIONS = ['md', 'txt', 'pdf', 'docx', 'json'];
|
|
16
|
+
const JSON_MAX_DEPTH = 12;
|
|
17
|
+
const JSON_MAX_ENTRIES = 5_000;
|
|
18
|
+
const JSON_MAX_ARRAY_ITEMS = 100;
|
|
19
|
+
const JSON_MAX_STRING_LENGTH = 500;
|
|
20
|
+
const JSON_MAX_TOP_LEVEL_KEYS = 25;
|
|
21
|
+
const SIMPLE_JSON_KEY_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
15
22
|
|
|
16
23
|
const USAGE = [
|
|
17
24
|
'Usage: notesia import <path> [options]',
|
|
18
25
|
'',
|
|
26
|
+
'Arguments:',
|
|
27
|
+
' <path> Path to directory to import (positional or --path flag)',
|
|
28
|
+
'',
|
|
19
29
|
'Options:',
|
|
20
|
-
' --
|
|
30
|
+
' --path <dir> Path to directory to import',
|
|
31
|
+
` --ext <list> Comma-separated extensions to import (default: ${DEFAULT_EXTENSIONS.join(',')})`,
|
|
21
32
|
' --folder <name> Target folder name in Notesia',
|
|
22
33
|
' --tags <list> Comma-separated tags to apply to all imported notes',
|
|
23
34
|
' --conflict <mode> Conflict resolution: skip | overwrite (default: skip)',
|
|
35
|
+
' --create-folder Create folder if it does not exist (default: yes)',
|
|
36
|
+
' --no-create-folder Do not create folder if it does not exist',
|
|
24
37
|
' --dry-run Preview without writing',
|
|
25
38
|
' --format <fmt> Output format: text | json (default: text)',
|
|
26
39
|
' --quiet Suppress stdout output',
|
|
@@ -38,28 +51,35 @@ export function parseImportArgs(argv) {
|
|
|
38
51
|
process.exit(0);
|
|
39
52
|
}
|
|
40
53
|
|
|
41
|
-
const importPath = args.shift();
|
|
42
|
-
if (!importPath || importPath.startsWith('--')) {
|
|
43
|
-
process.stderr.write('Error: <path> argument is required\n\n' + USAGE);
|
|
44
|
-
process.exit(2);
|
|
45
|
-
}
|
|
46
|
-
|
|
47
54
|
const opts = {
|
|
48
|
-
importPath:
|
|
49
|
-
ext: [
|
|
55
|
+
importPath: null,
|
|
56
|
+
ext: [...DEFAULT_EXTENSIONS],
|
|
50
57
|
folder: null,
|
|
51
58
|
tags: [],
|
|
52
59
|
conflict: 'skip',
|
|
60
|
+
createFolder: true, // Auto-create folder by default (prompt if not exists)
|
|
53
61
|
dryRun: false,
|
|
54
62
|
format: 'text',
|
|
55
63
|
quiet: false,
|
|
56
64
|
verbose: false,
|
|
57
65
|
};
|
|
58
66
|
|
|
67
|
+
// Handle positional <path> argument OR --path flag
|
|
68
|
+
const firstArg = args[0];
|
|
69
|
+
if (firstArg && !firstArg.startsWith('--')) {
|
|
70
|
+
// Positional path: notesia import ~/Downloads
|
|
71
|
+
opts.importPath = path.resolve(firstArg);
|
|
72
|
+
args.shift();
|
|
73
|
+
}
|
|
74
|
+
|
|
59
75
|
for (let i = 0; i < args.length; i++) {
|
|
60
76
|
const token = args[i];
|
|
61
77
|
|
|
62
|
-
if (token === '--
|
|
78
|
+
if (token === '--path') {
|
|
79
|
+
const val = args[++i];
|
|
80
|
+
if (!val || val.startsWith('--')) { process.stderr.write('Error: --path requires a value\n\n' + USAGE); process.exit(2); }
|
|
81
|
+
opts.importPath = path.resolve(val);
|
|
82
|
+
} else if (token === '--ext') {
|
|
63
83
|
const val = args[++i];
|
|
64
84
|
if (!val || val.startsWith('--')) { process.stderr.write('Error: --ext requires a value\n\n' + USAGE); process.exit(2); }
|
|
65
85
|
opts.ext = val.split(',').map((e) => e.trim().toLowerCase()).filter(Boolean);
|
|
@@ -75,6 +95,10 @@ export function parseImportArgs(argv) {
|
|
|
75
95
|
const val = args[++i];
|
|
76
96
|
if (val !== 'skip' && val !== 'overwrite') { process.stderr.write('Error: --conflict must be "skip" or "overwrite"\n\n' + USAGE); process.exit(2); }
|
|
77
97
|
opts.conflict = val;
|
|
98
|
+
} else if (token === '--create-folder') {
|
|
99
|
+
opts.createFolder = true;
|
|
100
|
+
} else if (token === '--no-create-folder') {
|
|
101
|
+
opts.createFolder = false;
|
|
78
102
|
} else if (token === '--dry-run') {
|
|
79
103
|
opts.dryRun = true;
|
|
80
104
|
} else if (token === '--format') {
|
|
@@ -91,6 +115,12 @@ export function parseImportArgs(argv) {
|
|
|
91
115
|
}
|
|
92
116
|
}
|
|
93
117
|
|
|
118
|
+
// Validate that a path was provided
|
|
119
|
+
if (!opts.importPath) {
|
|
120
|
+
process.stderr.write('Error: <path> argument is required (use positional argument or --path flag)\n\n' + USAGE);
|
|
121
|
+
process.exit(2);
|
|
122
|
+
}
|
|
123
|
+
|
|
94
124
|
return opts;
|
|
95
125
|
}
|
|
96
126
|
|
|
@@ -129,9 +159,19 @@ async function discoverFiles(rootPath, extensions) {
|
|
|
129
159
|
|
|
130
160
|
// ─── Content parsing ─────────────────────────────────────────────────────────
|
|
131
161
|
|
|
132
|
-
function titleFromPath(filePath) {
|
|
133
|
-
|
|
134
|
-
|
|
162
|
+
function titleFromPath(filePath, relativePath) {
|
|
163
|
+
// Use relative path to include folder structure in title
|
|
164
|
+
// e.g., "folder1/SKILL.md" → "folder1/SKILL"
|
|
165
|
+
const baseName = path.basename(relativePath, path.extname(relativePath));
|
|
166
|
+
const folderName = path.dirname(relativePath);
|
|
167
|
+
const stem = baseName.replace(/[-_]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
168
|
+
|
|
169
|
+
// If file is in a subfolder, prepend folder to title
|
|
170
|
+
if (folderName && folderName !== '.') {
|
|
171
|
+
const folderStem = folderName.replace(/[-_]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
172
|
+
return `${folderStem}/${stem}`;
|
|
173
|
+
}
|
|
174
|
+
return stem;
|
|
135
175
|
}
|
|
136
176
|
|
|
137
177
|
function stripHtml(html) {
|
|
@@ -143,8 +183,185 @@ function stripHtml(html) {
|
|
|
143
183
|
.trim();
|
|
144
184
|
}
|
|
145
185
|
|
|
146
|
-
|
|
147
|
-
|
|
186
|
+
function decodeXmlEntities(value) {
|
|
187
|
+
if (!value) return '';
|
|
188
|
+
return value
|
|
189
|
+
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
|
190
|
+
.replace(/&/g, '&')
|
|
191
|
+
.replace(/</g, '<')
|
|
192
|
+
.replace(/>/g, '>')
|
|
193
|
+
.replace(/"/g, '"')
|
|
194
|
+
.replace(/'/g, "'")
|
|
195
|
+
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
|
|
196
|
+
.replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10)))
|
|
197
|
+
.replace(/\u00A0/g, ' ');
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function sharedLeadingWhitespace(a, b) {
|
|
201
|
+
const max = Math.min(a.length, b.length);
|
|
202
|
+
let i = 0;
|
|
203
|
+
while (i < max && a[i] === b[i]) i++;
|
|
204
|
+
return a.slice(0, i);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function normalizeFencedCodeBlocks(markdown) {
|
|
208
|
+
if (!markdown || !markdown.includes('```')) return markdown;
|
|
209
|
+
|
|
210
|
+
return markdown.replace(/```([^\n]*)\n([\s\S]*?)\n```/g, (full, lang, body) => {
|
|
211
|
+
const lines = String(body).replace(/\u00A0/g, ' ').split('\n');
|
|
212
|
+
const nonEmpty = lines.filter((line) => line.trim().length > 0);
|
|
213
|
+
if (nonEmpty.length === 0) return full;
|
|
214
|
+
|
|
215
|
+
let commonIndent = null;
|
|
216
|
+
for (const line of nonEmpty) {
|
|
217
|
+
const indent = line.match(/^[ \t]*/)?.[0] || '';
|
|
218
|
+
if (!indent) {
|
|
219
|
+
commonIndent = '';
|
|
220
|
+
break;
|
|
221
|
+
}
|
|
222
|
+
commonIndent = commonIndent === null ? indent : sharedLeadingWhitespace(commonIndent, indent);
|
|
223
|
+
if (!commonIndent) break;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
if (!commonIndent) {
|
|
227
|
+
return `\`\`\`${lang}\n${lines.join('\n')}\n\`\`\``;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const dedented = lines.map((line) =>
|
|
231
|
+
line.startsWith(commonIndent) ? line.slice(commonIndent.length) : line
|
|
232
|
+
);
|
|
233
|
+
return `\`\`\`${lang}\n${dedented.join('\n')}\n\`\`\``;
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function appendJsonPath(basePath, key) {
|
|
238
|
+
if (SIMPLE_JSON_KEY_RE.test(key)) {
|
|
239
|
+
return basePath === '$' ? `$.${key}` : `${basePath}.${key}`;
|
|
240
|
+
}
|
|
241
|
+
const escapedKey = String(key).replaceAll('\\', '\\\\').replaceAll('"', '\\"');
|
|
242
|
+
return `${basePath}["${escapedKey}"]`;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function formatJsonValue(value) {
|
|
246
|
+
if (typeof value === 'string') {
|
|
247
|
+
const normalized = value.replace(/\s+/g, ' ').trim();
|
|
248
|
+
if (normalized.length > JSON_MAX_STRING_LENGTH) {
|
|
249
|
+
const truncatedChars = normalized.length - JSON_MAX_STRING_LENGTH;
|
|
250
|
+
const shortened = normalized.slice(0, JSON_MAX_STRING_LENGTH);
|
|
251
|
+
return `${JSON.stringify(shortened)} [truncated ${truncatedChars} chars]`;
|
|
252
|
+
}
|
|
253
|
+
return JSON.stringify(normalized);
|
|
254
|
+
}
|
|
255
|
+
if (value === null) return 'null';
|
|
256
|
+
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
|
257
|
+
return JSON.stringify(value);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function flattenJsonForEmbedding(root) {
|
|
261
|
+
const lines = [];
|
|
262
|
+
const notes = [];
|
|
263
|
+
const stack = [{ path: '$', value: root, depth: 0 }];
|
|
264
|
+
let hitEntryLimit = false;
|
|
265
|
+
|
|
266
|
+
while (stack.length > 0) {
|
|
267
|
+
if (lines.length >= JSON_MAX_ENTRIES) {
|
|
268
|
+
hitEntryLimit = true;
|
|
269
|
+
break;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const current = stack.pop();
|
|
273
|
+
const { path: currentPath, value, depth } = current;
|
|
274
|
+
|
|
275
|
+
if (value === null || typeof value !== 'object') {
|
|
276
|
+
lines.push(`${currentPath}: ${formatJsonValue(value)}`);
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
if (depth >= JSON_MAX_DEPTH) {
|
|
281
|
+
lines.push(`${currentPath}: [max depth ${JSON_MAX_DEPTH} reached]`);
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
if (Array.isArray(value)) {
|
|
286
|
+
lines.push(`${currentPath}: [array length=${value.length}]`);
|
|
287
|
+
const keptItems = Math.min(value.length, JSON_MAX_ARRAY_ITEMS);
|
|
288
|
+
|
|
289
|
+
for (let i = keptItems - 1; i >= 0; i -= 1) {
|
|
290
|
+
stack.push({
|
|
291
|
+
path: `${currentPath}[${i}]`,
|
|
292
|
+
value: value[i],
|
|
293
|
+
depth: depth + 1,
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
if (value.length > keptItems) {
|
|
298
|
+
lines.push(`${currentPath}: [omitted after item ${keptItems - 1}] (${value.length - keptItems} items omitted)`);
|
|
299
|
+
}
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const entries = Object.entries(value);
|
|
304
|
+
if (entries.length === 0) {
|
|
305
|
+
lines.push(`${currentPath}: {}`);
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
for (let i = entries.length - 1; i >= 0; i -= 1) {
|
|
310
|
+
const [key, childValue] = entries[i];
|
|
311
|
+
stack.push({
|
|
312
|
+
path: appendJsonPath(currentPath, key),
|
|
313
|
+
value: childValue,
|
|
314
|
+
depth: depth + 1,
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
if (hitEntryLimit) {
|
|
320
|
+
notes.push(`[truncated] Flattened field limit reached (${JSON_MAX_ENTRIES}); remaining fields omitted.`);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
return { lines, notes };
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function formatJsonForEmbedding(jsonValue, sourcePath = '') {
|
|
327
|
+
const rootType = Array.isArray(jsonValue)
|
|
328
|
+
? 'array'
|
|
329
|
+
: (jsonValue === null ? 'null' : typeof jsonValue);
|
|
330
|
+
const topLevelKeys = rootType === 'object' && jsonValue
|
|
331
|
+
? Object.keys(jsonValue)
|
|
332
|
+
: [];
|
|
333
|
+
const { lines, notes } = flattenJsonForEmbedding(jsonValue);
|
|
334
|
+
|
|
335
|
+
const output = [
|
|
336
|
+
'# JSON Import',
|
|
337
|
+
'',
|
|
338
|
+
`Source: ${sourcePath || 'unknown'}`,
|
|
339
|
+
`Root type: ${rootType}`,
|
|
340
|
+
];
|
|
341
|
+
|
|
342
|
+
if (rootType === 'array') {
|
|
343
|
+
output.push(`Top-level items: ${jsonValue.length}`);
|
|
344
|
+
} else if (rootType === 'object') {
|
|
345
|
+
const visibleKeys = topLevelKeys.slice(0, JSON_MAX_TOP_LEVEL_KEYS);
|
|
346
|
+
const suffix = topLevelKeys.length > visibleKeys.length
|
|
347
|
+
? ` (+${topLevelKeys.length - visibleKeys.length} more)`
|
|
348
|
+
: '';
|
|
349
|
+
output.push(`Top-level keys (${topLevelKeys.length}): ${visibleKeys.join(', ')}${suffix}`);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
output.push('', '## Flattened Fields', '');
|
|
353
|
+
output.push(...(lines.length > 0 ? lines : ['$: [empty document]']));
|
|
354
|
+
|
|
355
|
+
if (notes.length > 0) {
|
|
356
|
+
output.push('', '## Import Notes', '');
|
|
357
|
+
output.push(...notes.map((note) => `- ${note}`));
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
return output.join('\n').trim();
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
async function parseFile(absolutePath, extension, relativePath) {
|
|
364
|
+
const defaultTitle = titleFromPath(absolutePath, relativePath);
|
|
148
365
|
try {
|
|
149
366
|
if (extension === 'md' || extension === 'txt') {
|
|
150
367
|
const body = await fs.readFile(absolutePath, 'utf8');
|
|
@@ -156,23 +373,114 @@ async function parseFile(absolutePath, extension) {
|
|
|
156
373
|
const htmlTitle = titleMatch ? titleMatch[1].replace(/\s+/g, ' ').trim() : null;
|
|
157
374
|
return { title: htmlTitle || defaultTitle, body: stripHtml(raw) };
|
|
158
375
|
}
|
|
376
|
+
if (extension === 'json') {
|
|
377
|
+
const raw = await fs.readFile(absolutePath, 'utf8');
|
|
378
|
+
let parsedJson;
|
|
379
|
+
try {
|
|
380
|
+
parsedJson = JSON.parse(raw.replace(/^\uFEFF/, ''));
|
|
381
|
+
} catch (err) {
|
|
382
|
+
return { title: defaultTitle, body: '', error: `Invalid JSON: ${err.message}` };
|
|
383
|
+
}
|
|
384
|
+
return {
|
|
385
|
+
title: defaultTitle,
|
|
386
|
+
body: formatJsonForEmbedding(parsedJson, relativePath || path.basename(absolutePath)),
|
|
387
|
+
};
|
|
388
|
+
}
|
|
159
389
|
if (extension === 'pdf') {
|
|
160
|
-
//
|
|
390
|
+
// Use pdfjs-dist legacy build for Node.js PDF text extraction
|
|
161
391
|
try {
|
|
162
|
-
const
|
|
163
|
-
pdfjsLib.GlobalWorkerOptions.workerSrc = '';
|
|
392
|
+
const pdfjsLib = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
|
164
393
|
const data = await fs.readFile(absolutePath);
|
|
165
|
-
const pdf = await pdfjsLib.getDocument({ data: new Uint8Array(data)
|
|
394
|
+
const pdf = await pdfjsLib.getDocument({ data: new Uint8Array(data.buffer) }).promise;
|
|
166
395
|
const pages = [];
|
|
167
396
|
for (let i = 1; i <= pdf.numPages; i++) {
|
|
168
397
|
const page = await pdf.getPage(i);
|
|
169
398
|
const content = await page.getTextContent();
|
|
170
|
-
|
|
399
|
+
|
|
400
|
+
// Extract text with better structure preservation
|
|
401
|
+
const items = content.items;
|
|
402
|
+
let pageText = '';
|
|
403
|
+
let lastY = null;
|
|
404
|
+
|
|
405
|
+
for (const item of items) {
|
|
406
|
+
if (lastY !== null && Math.abs(item.transform[5] - lastY) > 5) {
|
|
407
|
+
// Different line
|
|
408
|
+
pageText += '\n';
|
|
409
|
+
} else if (pageText !== '') {
|
|
410
|
+
// Same line - add space between items
|
|
411
|
+
pageText += ' ';
|
|
412
|
+
}
|
|
413
|
+
pageText += item.str;
|
|
414
|
+
lastY = item.transform[5];
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
pages.push(pageText.trim());
|
|
171
418
|
page.cleanup();
|
|
172
419
|
}
|
|
420
|
+
|
|
173
421
|
let title = defaultTitle;
|
|
174
422
|
try { const meta = await pdf.getMetadata(); if (meta?.info?.Title?.trim()) title = meta.info.Title.trim(); } catch {}
|
|
175
|
-
|
|
423
|
+
|
|
424
|
+
// Join pages with double newline and clean up
|
|
425
|
+
const body = pages.join('\n\n')
|
|
426
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
427
|
+
.replace(/[ \t]+/g, ' ')
|
|
428
|
+
.trim();
|
|
429
|
+
|
|
430
|
+
return { title, body };
|
|
431
|
+
} catch (err) {
|
|
432
|
+
return { title: defaultTitle, body: '', error: err.message };
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
if (extension === 'docx') {
|
|
436
|
+
// Use JSZip for DOCX text extraction
|
|
437
|
+
try {
|
|
438
|
+
const JSZip = (await import('jszip')).default;
|
|
439
|
+
const data = await fs.readFile(absolutePath);
|
|
440
|
+
const zip = await JSZip.loadAsync(data);
|
|
441
|
+
const docXml = await zip.file('word/document.xml')?.async('string');
|
|
442
|
+
if (!docXml) {
|
|
443
|
+
return { title: defaultTitle, body: '', error: 'Invalid DOCX file: missing document.xml' };
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// Extract title from docProps/core.xml if available
|
|
447
|
+
let title = defaultTitle;
|
|
448
|
+
try {
|
|
449
|
+
const coreXml = await zip.file('docProps/core.xml')?.async('string');
|
|
450
|
+
if (coreXml) {
|
|
451
|
+
const titleMatch = coreXml.match(/<dc:title[^>]*>([^<]*)<\/dc:title>/);
|
|
452
|
+
if (titleMatch && titleMatch[1]?.trim()) {
|
|
453
|
+
title = titleMatch[1].trim();
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
} catch {}
|
|
457
|
+
|
|
458
|
+
// Parse XML to extract plain text with proper line breaks
|
|
459
|
+
let text = docXml;
|
|
460
|
+
|
|
461
|
+
// Preserve CDATA text content before stripping tags.
|
|
462
|
+
text = text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1');
|
|
463
|
+
|
|
464
|
+
// Handle paragraph spacing - use double newline for paragraphs
|
|
465
|
+
text = text
|
|
466
|
+
.replace(/<\/w:p>/g, '\n\n')
|
|
467
|
+
.replace(/<w:tab[^>]*\/>/g, '\t')
|
|
468
|
+
.replace(/<w:br[^>]*\/>/g, '\n')
|
|
469
|
+
.replace(/<w:cr[^>]*\/>/g, '\n');
|
|
470
|
+
|
|
471
|
+
// Remove all remaining XML tags
|
|
472
|
+
text = text
|
|
473
|
+
.replace(/<[^>]+>/g, '')
|
|
474
|
+
.replace(/\r\n/g, '\n')
|
|
475
|
+
.replace(/\r/g, '\n');
|
|
476
|
+
|
|
477
|
+
text = decodeXmlEntities(text)
|
|
478
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
479
|
+
.trim();
|
|
480
|
+
|
|
481
|
+
text = normalizeFencedCodeBlocks(text);
|
|
482
|
+
|
|
483
|
+
return { title, body: text };
|
|
176
484
|
} catch (err) {
|
|
177
485
|
return { title: defaultTitle, body: '', error: err.message };
|
|
178
486
|
}
|
|
@@ -194,8 +502,15 @@ function splitContent(title, body) {
|
|
|
194
502
|
if (remaining.length <= NOTE_CHAR_LIMIT) { chunks.push(remaining); break; }
|
|
195
503
|
const searchStart = Math.max(0, NOTE_CHAR_LIMIT - 500);
|
|
196
504
|
const slice = remaining.slice(searchStart, NOTE_CHAR_LIMIT);
|
|
197
|
-
const
|
|
198
|
-
|
|
505
|
+
const lastParagraphBreak = slice.lastIndexOf('\n\n');
|
|
506
|
+
if (lastParagraphBreak !== -1) {
|
|
507
|
+
chunks.push(remaining.slice(0, searchStart + lastParagraphBreak + 2));
|
|
508
|
+
remaining = remaining.slice(searchStart + lastParagraphBreak + 2);
|
|
509
|
+
continue;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
const lastLineBreak = slice.lastIndexOf('\n');
|
|
513
|
+
const splitAt = lastLineBreak !== -1 ? searchStart + lastLineBreak + 1 : NOTE_CHAR_LIMIT;
|
|
199
514
|
chunks.push(remaining.slice(0, splitAt));
|
|
200
515
|
remaining = remaining.slice(splitAt);
|
|
201
516
|
}
|
|
@@ -210,11 +525,60 @@ function printResult(result, opts) {
|
|
|
210
525
|
const { format = 'text', quiet = false } = opts;
|
|
211
526
|
if (quiet) return;
|
|
212
527
|
if (format === 'json') { process.stdout.write(JSON.stringify(result) + '\n'); return; }
|
|
213
|
-
|
|
528
|
+
|
|
529
|
+
const { created, skipped, overwritten, failed, elapsedMs, items = [] } = result;
|
|
214
530
|
const parts = [`${created} created`];
|
|
215
531
|
if (overwritten > 0) parts.push(`${overwritten} overwritten`);
|
|
216
532
|
parts.push(`${skipped} skipped`, `${failed} failed`);
|
|
217
533
|
process.stdout.write(`Import complete: ${parts.join(', ')} (${formatElapsed(elapsedMs)})\n`);
|
|
534
|
+
|
|
535
|
+
// Print summary table if there are items
|
|
536
|
+
if (items.length > 0) {
|
|
537
|
+
process.stdout.write('\n');
|
|
538
|
+
process.stdout.write('─'.repeat(80) + '\n');
|
|
539
|
+
process.stdout.write(`${'Status'.padEnd(12)} | ${'Source'.padEnd(50)} | Reason\n`);
|
|
540
|
+
process.stdout.write('─'.repeat(80) + '\n');
|
|
541
|
+
|
|
542
|
+
for (const item of items) {
|
|
543
|
+
const status = (item.status || 'unknown').padEnd(12);
|
|
544
|
+
const source = (item.source_path || '').slice(0, 48).padEnd(50);
|
|
545
|
+
let reason = '';
|
|
546
|
+
|
|
547
|
+
if (item.status === 'created') {
|
|
548
|
+
reason = `noteId: ${item.noteId}`;
|
|
549
|
+
} else if (item.status === 'skipped') {
|
|
550
|
+
reason = `Note with same title already exists in target folder (noteId: ${item.noteId})`;
|
|
551
|
+
} else if (item.status === 'overwritten') {
|
|
552
|
+
reason = `noteId: ${item.noteId}`;
|
|
553
|
+
} else if (item.status === 'failed') {
|
|
554
|
+
reason = item.error || 'Unknown error';
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
process.stdout.write(`${status} | ${source} | ${reason}\n`);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
process.stdout.write('─'.repeat(80) + '\n');
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// ─── Interactive prompt ──────────────────────────────────────────────────────
|
|
565
|
+
|
|
566
|
+
function createInterface() {
|
|
567
|
+
return readline.createInterface({
|
|
568
|
+
input: process.stdin,
|
|
569
|
+
output: process.stdout,
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
async function promptYesNo(question) {
|
|
574
|
+
return new Promise((resolve) => {
|
|
575
|
+
const rl = createInterface();
|
|
576
|
+
rl.question(question, (answer) => {
|
|
577
|
+
rl.close();
|
|
578
|
+
const normalized = answer.trim().toLowerCase();
|
|
579
|
+
resolve(normalized === 'y' || normalized === 'yes');
|
|
580
|
+
});
|
|
581
|
+
});
|
|
218
582
|
}
|
|
219
583
|
|
|
220
584
|
// ─── Main entry point ────────────────────────────────────────────────────────
|
|
@@ -242,7 +606,7 @@ export async function runImport(argv, deps = {}) {
|
|
|
242
606
|
for (const file of files) {
|
|
243
607
|
if (opts.verbose) process.stderr.write(` [parsing] ${file.relativePath}\n`);
|
|
244
608
|
|
|
245
|
-
const parsed = await parseFile(file.absolutePath, file.extension);
|
|
609
|
+
const parsed = await parseFile(file.absolutePath, file.extension, file.relativePath);
|
|
246
610
|
|
|
247
611
|
if (parsed.error && !parsed.body) {
|
|
248
612
|
if (opts.verbose) process.stderr.write(` [failed] ${file.relativePath} — ${parsed.error}\n`);
|
|
@@ -268,9 +632,11 @@ export async function runImport(argv, deps = {}) {
|
|
|
268
632
|
source_path: item.source_path,
|
|
269
633
|
_parseError: item._parseError || null,
|
|
270
634
|
})),
|
|
271
|
-
folder: opts.folder,
|
|
272
|
-
tags: opts.tags,
|
|
273
|
-
conflict: opts.conflict,
|
|
635
|
+
folder: opts.folder || null,
|
|
636
|
+
tags: opts.tags || [],
|
|
637
|
+
conflict: opts.conflict || 'skip',
|
|
638
|
+
// Always check first - we'll prompt if needed
|
|
639
|
+
create_folder_if_missing: false,
|
|
274
640
|
dry_run: opts.dryRun,
|
|
275
641
|
};
|
|
276
642
|
|
|
@@ -279,7 +645,14 @@ export async function runImport(argv, deps = {}) {
|
|
|
279
645
|
|
|
280
646
|
let response;
|
|
281
647
|
try {
|
|
648
|
+
if (opts.verbose) {
|
|
649
|
+
process.stderr.write(` [debug] calling bridge with ${items.length} items, folder: "${opts.folder}"\n`);
|
|
650
|
+
}
|
|
282
651
|
response = await bridge({ op: 'bulk_import', payload }, { timeoutMs });
|
|
652
|
+
process.stderr.write(` [debug] bridge response ok: ${response?.ok}, hasData: ${!!response?.data}\n`);
|
|
653
|
+
if (opts.verbose || !response?.ok) {
|
|
654
|
+
process.stderr.write(` [debug] full response: ${JSON.stringify(response)}\n`);
|
|
655
|
+
}
|
|
283
656
|
} catch (err) {
|
|
284
657
|
const code = err.code || 'BRIDGE_ERROR';
|
|
285
658
|
if (code === 'NOTESIA_APP_NOT_RUNNING') {
|
|
@@ -290,14 +663,59 @@ export async function runImport(argv, deps = {}) {
|
|
|
290
663
|
process.exit(1);
|
|
291
664
|
}
|
|
292
665
|
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
666
|
+
// Check for errors - could be at top level or nested in data
|
|
667
|
+
const hasError = !response || response.ok === false || (response.data && response.data.ok === false);
|
|
668
|
+
if (hasError) {
|
|
669
|
+
const code = response?.error?.code || response?.data?.error?.code || 'BRIDGE_ERROR';
|
|
670
|
+
const message = response?.error?.message || response?.data?.error?.message || 'Bridge request failed';
|
|
671
|
+
|
|
672
|
+
// Handle folder not found - prompt user to create (only if --create-folder is set)
|
|
673
|
+
const errorData = response?.error || response?.data?.error;
|
|
674
|
+
if (code === 'FOLDER_NOT_FOUND' && opts.folder && opts.createFolder) {
|
|
675
|
+
const folderName = errorData?.folderName || opts.folder;
|
|
676
|
+
process.stdout.write(`\nFolder "${folderName}" does not exist in Notesia.\n`);
|
|
677
|
+
const shouldCreate = await promptYesNo('Do you want to create it and continue import? [y/N] ');
|
|
678
|
+
if (shouldCreate) {
|
|
679
|
+
// Retry with create_folder_if_missing = true
|
|
680
|
+
const retryPayload = { ...payload, create_folder_if_missing: true };
|
|
681
|
+
response = await bridge({ op: 'bulk_import', payload: retryPayload }, { timeoutMs });
|
|
682
|
+
// Check both top-level ok and nested data.ok
|
|
683
|
+
const retryOk = response?.ok === true && (!response.data || response.data.ok !== false);
|
|
684
|
+
if (!retryOk) {
|
|
685
|
+
const retryCode = response?.error?.code || response?.data?.error?.code || 'BRIDGE_ERROR';
|
|
686
|
+
const retryMessage = response?.error?.message || response?.data?.error?.message || 'Bridge request failed';
|
|
687
|
+
process.stderr.write(`[${retryCode}] ${retryMessage}\n`);
|
|
688
|
+
process.exit(1);
|
|
689
|
+
}
|
|
690
|
+
// Success - continue with response.data as the result
|
|
691
|
+
} else {
|
|
692
|
+
process.stderr.write('Import cancelled.\n');
|
|
693
|
+
process.exit(1);
|
|
694
|
+
}
|
|
695
|
+
} else if (code === 'FOLDER_NOT_FOUND' && opts.folder && !opts.createFolder) {
|
|
696
|
+
// User specified --no-create-folder, so just show the error
|
|
697
|
+
process.stderr.write(`[${code}] ${message}\n`);
|
|
698
|
+
process.exit(1);
|
|
699
|
+
} else {
|
|
700
|
+
process.stderr.write(`[${code}] ${message}\n`);
|
|
701
|
+
process.exit(1);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// Check if we have valid data in the response
|
|
706
|
+
if (!response.data) {
|
|
707
|
+
process.stderr.write(`[BRIDGE_ERROR] No data in bridge response. Raw response: ${JSON.stringify(response)}\n`);
|
|
297
708
|
process.exit(1);
|
|
298
709
|
}
|
|
299
710
|
|
|
300
|
-
const result = {
|
|
711
|
+
const result = {
|
|
712
|
+
created: response.data?.created ?? 0,
|
|
713
|
+
skipped: response.data?.skipped ?? 0,
|
|
714
|
+
overwritten: response.data?.overwritten ?? 0,
|
|
715
|
+
failed: response.data?.failed ?? 0,
|
|
716
|
+
elapsedMs: response.data?.elapsedMs ?? (Date.now() - startMs),
|
|
717
|
+
items: response.data?.items ?? [],
|
|
718
|
+
};
|
|
301
719
|
printResult(result, { format: opts.format, quiet: opts.quiet });
|
|
302
720
|
process.exit(result.failed === 0 ? 0 : 1);
|
|
303
721
|
}
|
package/src/io.mjs
CHANGED
|
@@ -170,7 +170,9 @@ export function printKbResult(op, result, {
|
|
|
170
170
|
if (citations.length > 0) {
|
|
171
171
|
writeStdout('\n\nSources:\n');
|
|
172
172
|
for (const c of citations) {
|
|
173
|
-
|
|
173
|
+
const sourceType = c.type ? `[${c.type.toUpperCase()}] ` : '';
|
|
174
|
+
const id = c.noteId ?? c.todoId ?? 'unknown';
|
|
175
|
+
writeStdout(` - ${sourceType}[${id}] ${c.title}\n`);
|
|
174
176
|
}
|
|
175
177
|
} else {
|
|
176
178
|
writeStdout('\n');
|