@clarionstack/notesia 0.0.9 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands.mjs +28 -0
- package/src/import.mjs +116 -8
- package/src/io.mjs +4 -3
package/package.json
CHANGED
package/src/commands.mjs
CHANGED
|
@@ -103,6 +103,25 @@ export async function defaultResolveKbConversationId(explicitConversationId = nu
|
|
|
103
103
|
}
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
+
export async function defaultPersistKbConversationId(conversationId) {
|
|
107
|
+
if (!UUID_REGEX.test(conversationId || '')) {
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const nowMs = Date.now();
|
|
112
|
+
const sessionFilePath = getKbSessionStorePath();
|
|
113
|
+
const sessionKey = getTerminalSessionKey();
|
|
114
|
+
|
|
115
|
+
try {
|
|
116
|
+
const loaded = await loadKbSessionStore(sessionFilePath);
|
|
117
|
+
const store = pruneKbSessionStore(loaded, nowMs);
|
|
118
|
+
store[sessionKey] = { conversationId, updatedAt: nowMs };
|
|
119
|
+
await saveKbSessionStore(sessionFilePath, store);
|
|
120
|
+
} catch {
|
|
121
|
+
// Ignore persistence failures; kb ask can still continue statelessly.
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
106
125
|
export class CliCommandError extends Error {
|
|
107
126
|
constructor(code, message) {
|
|
108
127
|
super(message);
|
|
@@ -451,6 +470,7 @@ export async function runCli(argv = [], {
|
|
|
451
470
|
readInputContent = defaultReadInputContent,
|
|
452
471
|
writeStdout = (chunk) => process.stdout.write(chunk),
|
|
453
472
|
resolveKbConversationId = defaultResolveKbConversationId,
|
|
473
|
+
persistKbConversationId = defaultPersistKbConversationId,
|
|
454
474
|
} = {}) {
|
|
455
475
|
const parsed = parseArgs(argv);
|
|
456
476
|
|
|
@@ -482,6 +502,14 @@ export async function runCli(argv = [], {
|
|
|
482
502
|
throw new CliCommandError(code, message);
|
|
483
503
|
}
|
|
484
504
|
|
|
505
|
+
if (
|
|
506
|
+
parsed.op === 'kb-ask'
|
|
507
|
+
&& typeof persistKbConversationId === 'function'
|
|
508
|
+
&& UUID_REGEX.test(response.data?.conversation_id || '')
|
|
509
|
+
) {
|
|
510
|
+
await persistKbConversationId(response.data.conversation_id);
|
|
511
|
+
}
|
|
512
|
+
|
|
485
513
|
if (parsed.op === 'get') {
|
|
486
514
|
printRetrieveResult(response.data, {
|
|
487
515
|
format: parsed.format,
|
package/src/import.mjs
CHANGED
|
@@ -18,8 +18,11 @@ const DEFAULT_EXTENSIONS = ['md', 'txt', 'pdf', 'docx', 'json'];
|
|
|
18
18
|
const JSON_MAX_DEPTH = 12;
|
|
19
19
|
const JSON_MAX_ENTRIES = 5_000;
|
|
20
20
|
const JSON_MAX_ARRAY_ITEMS = 100;
|
|
21
|
+
const JSON_MAX_OBJECT_KEYS = 200;
|
|
21
22
|
const JSON_MAX_STRING_LENGTH = 500;
|
|
22
23
|
const JSON_MAX_TOP_LEVEL_KEYS = 25;
|
|
24
|
+
const JSON_PREVIEW_BYTES = 64 * 1024;
|
|
25
|
+
const JSON_DEFAULT_MAX_PARSE_MB = 8;
|
|
23
26
|
const SIMPLE_JSON_KEY_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
|
|
24
27
|
|
|
25
28
|
const USAGE = [
|
|
@@ -236,6 +239,61 @@ function normalizeFencedCodeBlocks(markdown) {
|
|
|
236
239
|
});
|
|
237
240
|
}
|
|
238
241
|
|
|
242
|
+
function getJsonMaxParseBytes() {
|
|
243
|
+
const rawLimit = Number(process.env.NOTESIA_CLI_JSON_MAX_PARSE_MB);
|
|
244
|
+
if (Number.isFinite(rawLimit) && rawLimit > 0) {
|
|
245
|
+
return Math.floor(rawLimit * 1024 * 1024);
|
|
246
|
+
}
|
|
247
|
+
return JSON_DEFAULT_MAX_PARSE_MB * 1024 * 1024;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function formatByteSize(bytes) {
|
|
251
|
+
if (!Number.isFinite(bytes) || bytes < 0) return 'unknown';
|
|
252
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
253
|
+
const kb = bytes / 1024;
|
|
254
|
+
if (kb < 1024) return `${kb.toFixed(1)} KB`;
|
|
255
|
+
const mb = kb / 1024;
|
|
256
|
+
return `${mb.toFixed(2)} MB`;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
async function readFilePreview(filePath, maxBytes) {
|
|
260
|
+
const handle = await fs.open(filePath, 'r');
|
|
261
|
+
try {
|
|
262
|
+
const buffer = Buffer.alloc(Math.max(1, maxBytes));
|
|
263
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
|
264
|
+
return buffer.subarray(0, bytesRead).toString('utf8').replace(/\u0000/g, '');
|
|
265
|
+
} finally {
|
|
266
|
+
await handle.close();
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function formatOversizedJsonImport({
|
|
271
|
+
sourcePath = '',
|
|
272
|
+
fileSizeBytes = 0,
|
|
273
|
+
parseLimitBytes = getJsonMaxParseBytes(),
|
|
274
|
+
previewText = '',
|
|
275
|
+
}) {
|
|
276
|
+
const cleanedPreview = String(previewText || '').trim() || '[empty preview]';
|
|
277
|
+
return [
|
|
278
|
+
'# JSON Import',
|
|
279
|
+
'',
|
|
280
|
+
`Source: ${sourcePath || 'unknown'}`,
|
|
281
|
+
'Root type: unknown (skipped full parse)',
|
|
282
|
+
`File size: ${formatByteSize(fileSizeBytes)}`,
|
|
283
|
+
'',
|
|
284
|
+
'## Import Notes',
|
|
285
|
+
'',
|
|
286
|
+
`- [large-file] Full JSON parse skipped (${formatByteSize(fileSizeBytes)} exceeds limit ${formatByteSize(parseLimitBytes)}).`,
|
|
287
|
+
'- Set NOTESIA_CLI_JSON_MAX_PARSE_MB to raise the parse threshold when needed.',
|
|
288
|
+
'',
|
|
289
|
+
`## Raw Preview (first ${formatByteSize(Math.min(fileSizeBytes, JSON_PREVIEW_BYTES))})`,
|
|
290
|
+
'',
|
|
291
|
+
'~~~json',
|
|
292
|
+
cleanedPreview,
|
|
293
|
+
'~~~',
|
|
294
|
+
].join('\n').trim();
|
|
295
|
+
}
|
|
296
|
+
|
|
239
297
|
function appendJsonPath(basePath, key) {
|
|
240
298
|
if (SIMPLE_JSON_KEY_RE.test(key)) {
|
|
241
299
|
return basePath === '$' ? `$.${key}` : `${basePath}.${key}`;
|
|
@@ -244,10 +302,11 @@ function appendJsonPath(basePath, key) {
|
|
|
244
302
|
return `${basePath}["${escapedKey}"]`;
|
|
245
303
|
}
|
|
246
304
|
|
|
247
|
-
function formatJsonValue(value) {
|
|
305
|
+
function formatJsonValue(value, metrics) {
|
|
248
306
|
if (typeof value === 'string') {
|
|
249
307
|
const normalized = value.replace(/\s+/g, ' ').trim();
|
|
250
308
|
if (normalized.length > JSON_MAX_STRING_LENGTH) {
|
|
309
|
+
metrics.truncatedStrings += 1;
|
|
251
310
|
const truncatedChars = normalized.length - JSON_MAX_STRING_LENGTH;
|
|
252
311
|
const shortened = normalized.slice(0, JSON_MAX_STRING_LENGTH);
|
|
253
312
|
return `${JSON.stringify(shortened)} [truncated ${truncatedChars} chars]`;
|
|
@@ -263,11 +322,19 @@ function flattenJsonForEmbedding(root) {
|
|
|
263
322
|
const lines = [];
|
|
264
323
|
const notes = [];
|
|
265
324
|
const stack = [{ path: '$', value: root, depth: 0 }];
|
|
266
|
-
|
|
325
|
+
const metrics = {
|
|
326
|
+
hitEntryLimit: false,
|
|
327
|
+
depthTruncations: 0,
|
|
328
|
+
arraysTruncated: 0,
|
|
329
|
+
arrayItemsOmitted: 0,
|
|
330
|
+
objectsTruncated: 0,
|
|
331
|
+
objectKeysOmitted: 0,
|
|
332
|
+
truncatedStrings: 0,
|
|
333
|
+
};
|
|
267
334
|
|
|
268
335
|
while (stack.length > 0) {
|
|
269
336
|
if (lines.length >= JSON_MAX_ENTRIES) {
|
|
270
|
-
hitEntryLimit = true;
|
|
337
|
+
metrics.hitEntryLimit = true;
|
|
271
338
|
break;
|
|
272
339
|
}
|
|
273
340
|
|
|
@@ -275,11 +342,12 @@ function flattenJsonForEmbedding(root) {
|
|
|
275
342
|
const { path: currentPath, value, depth } = current;
|
|
276
343
|
|
|
277
344
|
if (value === null || typeof value !== 'object') {
|
|
278
|
-
lines.push(`${currentPath}: ${formatJsonValue(value)}`);
|
|
345
|
+
lines.push(`${currentPath}: ${formatJsonValue(value, metrics)}`);
|
|
279
346
|
continue;
|
|
280
347
|
}
|
|
281
348
|
|
|
282
349
|
if (depth >= JSON_MAX_DEPTH) {
|
|
350
|
+
metrics.depthTruncations += 1;
|
|
283
351
|
lines.push(`${currentPath}: [max depth ${JSON_MAX_DEPTH} reached]`);
|
|
284
352
|
continue;
|
|
285
353
|
}
|
|
@@ -297,6 +365,8 @@ function flattenJsonForEmbedding(root) {
|
|
|
297
365
|
}
|
|
298
366
|
|
|
299
367
|
if (value.length > keptItems) {
|
|
368
|
+
metrics.arraysTruncated += 1;
|
|
369
|
+
metrics.arrayItemsOmitted += value.length - keptItems;
|
|
300
370
|
lines.push(`${currentPath}: [omitted after item ${keptItems - 1}] (${value.length - keptItems} items omitted)`);
|
|
301
371
|
}
|
|
302
372
|
continue;
|
|
@@ -308,8 +378,17 @@ function flattenJsonForEmbedding(root) {
|
|
|
308
378
|
continue;
|
|
309
379
|
}
|
|
310
380
|
|
|
311
|
-
|
|
312
|
-
|
|
381
|
+
const keptEntries = entries.slice(0, JSON_MAX_OBJECT_KEYS);
|
|
382
|
+
if (entries.length > keptEntries.length) {
|
|
383
|
+
const omittedCount = entries.length - keptEntries.length;
|
|
384
|
+
metrics.objectsTruncated += 1;
|
|
385
|
+
metrics.objectKeysOmitted += omittedCount;
|
|
386
|
+
lines.push(`${currentPath}: [object keys=${entries.length}]`);
|
|
387
|
+
lines.push(`${currentPath}: [omitted ${omittedCount} keys after first ${keptEntries.length}]`);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
for (let i = keptEntries.length - 1; i >= 0; i -= 1) {
|
|
391
|
+
const [key, childValue] = keptEntries[i];
|
|
313
392
|
stack.push({
|
|
314
393
|
path: appendJsonPath(currentPath, key),
|
|
315
394
|
value: childValue,
|
|
@@ -318,9 +397,21 @@ function flattenJsonForEmbedding(root) {
|
|
|
318
397
|
}
|
|
319
398
|
}
|
|
320
399
|
|
|
321
|
-
if (hitEntryLimit) {
|
|
400
|
+
if (metrics.hitEntryLimit) {
|
|
322
401
|
notes.push(`[truncated] Flattened field limit reached (${JSON_MAX_ENTRIES}); remaining fields omitted.`);
|
|
323
402
|
}
|
|
403
|
+
if (metrics.depthTruncations > 0) {
|
|
404
|
+
notes.push(`[depth] Max-depth limit reached at ${metrics.depthTruncations} path(s) (limit=${JSON_MAX_DEPTH}).`);
|
|
405
|
+
}
|
|
406
|
+
if (metrics.arraysTruncated > 0) {
|
|
407
|
+
notes.push(`[arrays] Truncated ${metrics.arraysTruncated} array(s); ${metrics.arrayItemsOmitted} item(s) omitted beyond index ${JSON_MAX_ARRAY_ITEMS - 1}.`);
|
|
408
|
+
}
|
|
409
|
+
if (metrics.objectsTruncated > 0) {
|
|
410
|
+
notes.push(`[objects] Truncated ${metrics.objectsTruncated} object(s); ${metrics.objectKeysOmitted} key(s) omitted beyond first ${JSON_MAX_OBJECT_KEYS}.`);
|
|
411
|
+
}
|
|
412
|
+
if (metrics.truncatedStrings > 0) {
|
|
413
|
+
notes.push(`[strings] Truncated ${metrics.truncatedStrings} long string value(s) to ${JSON_MAX_STRING_LENGTH} chars each.`);
|
|
414
|
+
}
|
|
324
415
|
|
|
325
416
|
return { lines, notes };
|
|
326
417
|
}
|
|
@@ -376,6 +467,23 @@ async function parseFile(absolutePath, extension, relativePath) {
|
|
|
376
467
|
return { title: htmlTitle || defaultTitle, body: stripHtml(raw) };
|
|
377
468
|
}
|
|
378
469
|
if (extension === 'json') {
|
|
470
|
+
const sourcePath = relativePath || path.basename(absolutePath);
|
|
471
|
+
const stat = await fs.stat(absolutePath).catch(() => null);
|
|
472
|
+
const jsonMaxParseBytes = getJsonMaxParseBytes();
|
|
473
|
+
|
|
474
|
+
if (stat && stat.size > jsonMaxParseBytes) {
|
|
475
|
+
const preview = await readFilePreview(absolutePath, JSON_PREVIEW_BYTES).catch(() => '');
|
|
476
|
+
return {
|
|
477
|
+
title: defaultTitle,
|
|
478
|
+
body: formatOversizedJsonImport({
|
|
479
|
+
sourcePath,
|
|
480
|
+
fileSizeBytes: stat.size,
|
|
481
|
+
parseLimitBytes: jsonMaxParseBytes,
|
|
482
|
+
previewText: preview,
|
|
483
|
+
}),
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
|
|
379
487
|
const raw = await fs.readFile(absolutePath, 'utf8');
|
|
380
488
|
let parsedJson;
|
|
381
489
|
try {
|
|
@@ -385,7 +493,7 @@ async function parseFile(absolutePath, extension, relativePath) {
|
|
|
385
493
|
}
|
|
386
494
|
return {
|
|
387
495
|
title: defaultTitle,
|
|
388
|
-
body: formatJsonForEmbedding(parsedJson,
|
|
496
|
+
body: formatJsonForEmbedding(parsedJson, sourcePath),
|
|
389
497
|
};
|
|
390
498
|
}
|
|
391
499
|
if (extension === 'pdf') {
|
package/src/io.mjs
CHANGED
|
@@ -164,14 +164,15 @@ export function printKbResult(op, result, {
|
|
|
164
164
|
}
|
|
165
165
|
|
|
166
166
|
if (op === 'kb-ask') {
|
|
167
|
-
const answer = result?.answer || '';
|
|
167
|
+
const answer = result?.answer || result?.message?.text || '';
|
|
168
168
|
const citations = Array.isArray(result?.citations) ? result.citations : [];
|
|
169
169
|
writeStdout(answer);
|
|
170
170
|
if (citations.length > 0) {
|
|
171
171
|
writeStdout('\n\nSources:\n');
|
|
172
172
|
for (const c of citations) {
|
|
173
|
-
const
|
|
174
|
-
const
|
|
173
|
+
const citationType = c.type || c.entity_type || null;
|
|
174
|
+
const sourceType = citationType ? `[${String(citationType).toUpperCase()}] ` : '';
|
|
175
|
+
const id = c.noteId ?? c.todoId ?? c.entity_id ?? 'unknown';
|
|
175
176
|
writeStdout(` - ${sourceType}[${id}] ${c.title}\n`);
|
|
176
177
|
}
|
|
177
178
|
} else {
|