@karmaniverous/jeeves-watcher 0.6.8 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +243 -78
- package/dist/cli/jeeves-watcher/index.js +375 -100
- package/dist/index.d.ts +30 -0
- package/dist/index.js +375 -103
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import { resolve, extname, basename, dirname, isAbsolute, join, relative } from 'node:path';
|
|
1
2
|
import Fastify from 'fastify';
|
|
2
3
|
import { readFileSync, statSync, existsSync, mkdirSync, writeFileSync, readdirSync } from 'node:fs';
|
|
3
|
-
import { resolve, extname, basename, dirname, join, relative } from 'node:path';
|
|
4
4
|
import { JsonMap, jsonMapMapSchema } from '@karmaniverous/jsonmap';
|
|
5
5
|
import { pathToFileURL } from 'node:url';
|
|
6
6
|
import Handlebars from 'handlebars';
|
|
@@ -11,6 +11,7 @@ import { toMarkdown } from 'mdast-util-to-markdown';
|
|
|
11
11
|
import { capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
|
|
12
12
|
import rehypeParse from 'rehype-parse';
|
|
13
13
|
import { unified } from 'unified';
|
|
14
|
+
import yaml from 'js-yaml';
|
|
14
15
|
import Ajv from 'ajv';
|
|
15
16
|
import addFormats from 'ajv-formats';
|
|
16
17
|
import picomatch from 'picomatch';
|
|
@@ -18,13 +19,13 @@ import { readdir, stat, writeFile, rm, readFile, mkdir } from 'node:fs/promises'
|
|
|
18
19
|
import { z, ZodError } from 'zod';
|
|
19
20
|
import { JSONPath } from 'jsonpath-plus';
|
|
20
21
|
import { createHash } from 'node:crypto';
|
|
22
|
+
import crypto from 'crypto';
|
|
21
23
|
import chokidar from 'chokidar';
|
|
22
24
|
import { cosmiconfig } from 'cosmiconfig';
|
|
23
25
|
import { GoogleGenerativeAIEmbeddings } from '@langchain/google-genai';
|
|
24
26
|
import pino from 'pino';
|
|
25
27
|
import { v5 } from 'uuid';
|
|
26
28
|
import * as cheerio from 'cheerio';
|
|
27
|
-
import yaml from 'js-yaml';
|
|
28
29
|
import mammoth from 'mammoth';
|
|
29
30
|
import { MarkdownTextSplitter, RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
|
|
30
31
|
import { QdrantClient } from '@qdrant/js-client-rest';
|
|
@@ -334,6 +335,146 @@ async function buildTemplateEngine(rules, namedTemplates, templateHelpers, confi
|
|
|
334
335
|
return engine;
|
|
335
336
|
}
|
|
336
337
|
|
|
338
|
+
/**
|
|
339
|
+
* @module templates/rebaseHeadings
|
|
340
|
+
* Rebase Markdown heading levels so the minimum heading depth starts at baseHeading+1.
|
|
341
|
+
*/
|
|
342
|
+
/**
|
|
343
|
+
* Rebase Markdown headings.
|
|
344
|
+
*
|
|
345
|
+
* Finds all ATX headings (lines starting with one or more '#'),
|
|
346
|
+
* computes the minimum heading level present, and shifts all headings
|
|
347
|
+
* so that the minimum becomes baseHeading+1.
|
|
348
|
+
*/
|
|
349
|
+
function rebaseHeadings(markdown, baseHeading) {
|
|
350
|
+
if (!markdown.trim())
|
|
351
|
+
return markdown;
|
|
352
|
+
const lines = markdown.split(/\r?\n/);
|
|
353
|
+
const headingLevels = [];
|
|
354
|
+
for (const line of lines) {
|
|
355
|
+
const m = line.match(/^(#{1,6})\s+/);
|
|
356
|
+
if (m)
|
|
357
|
+
headingLevels.push(m[1].length);
|
|
358
|
+
}
|
|
359
|
+
if (headingLevels.length === 0)
|
|
360
|
+
return markdown;
|
|
361
|
+
const minLevel = Math.min(...headingLevels);
|
|
362
|
+
const targetMin = Math.min(6, Math.max(1, baseHeading + 1));
|
|
363
|
+
const delta = targetMin - minLevel;
|
|
364
|
+
if (delta === 0)
|
|
365
|
+
return markdown;
|
|
366
|
+
const rebased = lines.map((line) => {
|
|
367
|
+
const m = line.match(/^(#{1,6})(\s+.*)$/);
|
|
368
|
+
if (!m)
|
|
369
|
+
return line;
|
|
370
|
+
const level = m[1].length;
|
|
371
|
+
const newLevel = Math.min(6, Math.max(1, level + delta));
|
|
372
|
+
return `${'#'.repeat(newLevel)}${m[2]}`;
|
|
373
|
+
});
|
|
374
|
+
return rebased.join('\n');
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* @module templates/renderDoc
|
|
379
|
+
* Declarative renderer for YAML frontmatter + structured Markdown body.
|
|
380
|
+
*/
|
|
381
|
+
function renderSectionHeading(section, hbs, context = {}) {
|
|
382
|
+
const level = Math.min(6, Math.max(1, section.heading));
|
|
383
|
+
const label = section.label
|
|
384
|
+
? hbs.compile(section.label)(context)
|
|
385
|
+
: title(section.path);
|
|
386
|
+
return `${'#'.repeat(level)} ${label}`;
|
|
387
|
+
}
|
|
388
|
+
function callFormatHelper(hbs, helperName, value, args) {
|
|
389
|
+
const helper = hbs.helpers[helperName];
|
|
390
|
+
if (typeof helper !== 'function') {
|
|
391
|
+
throw new Error(`Format helper not found: ${helperName}`);
|
|
392
|
+
}
|
|
393
|
+
// Handlebars helpers may return SafeString or other values; coerce to string
|
|
394
|
+
const safeArgs = args ?? [];
|
|
395
|
+
const result = helper(value, ...safeArgs);
|
|
396
|
+
return typeof result === 'string' ? result : String(result);
|
|
397
|
+
}
|
|
398
|
+
function renderValueAsMarkdown(hbs, section, value) {
|
|
399
|
+
if (value === null || value === undefined)
|
|
400
|
+
return '';
|
|
401
|
+
if (section.format) {
|
|
402
|
+
const md = callFormatHelper(hbs, section.format, value, section.formatArgs);
|
|
403
|
+
return rebaseHeadings(md, section.heading);
|
|
404
|
+
}
|
|
405
|
+
const strValue = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
|
|
406
|
+
return hbs.Utils.escapeExpression(strValue);
|
|
407
|
+
}
|
|
408
|
+
function renderEach(hbs, section, value) {
|
|
409
|
+
if (!Array.isArray(value))
|
|
410
|
+
return '';
|
|
411
|
+
const items = [...value];
|
|
412
|
+
if (section.sort) {
|
|
413
|
+
items.sort((a, b) => {
|
|
414
|
+
const av = get(a, section.sort);
|
|
415
|
+
const bv = get(b, section.sort);
|
|
416
|
+
const aStr = typeof av === 'string' ? av : JSON.stringify(av ?? '');
|
|
417
|
+
const bStr = typeof bv === 'string' ? bv : JSON.stringify(bv ?? '');
|
|
418
|
+
return aStr.localeCompare(bStr);
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
const subHeadingLevel = Math.min(6, section.heading + 1);
|
|
422
|
+
const headingTpl = section.headingTemplate
|
|
423
|
+
? hbs.compile(section.headingTemplate)
|
|
424
|
+
: undefined;
|
|
425
|
+
const parts = [];
|
|
426
|
+
for (const item of items) {
|
|
427
|
+
const headingText = headingTpl
|
|
428
|
+
? headingTpl(item)
|
|
429
|
+
: '';
|
|
430
|
+
if (headingText) {
|
|
431
|
+
parts.push(`${'#'.repeat(subHeadingLevel)} ${headingText}`);
|
|
432
|
+
}
|
|
433
|
+
const contentVal = section.contentPath
|
|
434
|
+
? get(item, section.contentPath)
|
|
435
|
+
: item;
|
|
436
|
+
const md = renderValueAsMarkdown(hbs, { ...section, heading: subHeadingLevel }, contentVal);
|
|
437
|
+
if (md.trim())
|
|
438
|
+
parts.push(md.trim());
|
|
439
|
+
}
|
|
440
|
+
return parts.join('\n\n');
|
|
441
|
+
}
|
|
442
|
+
/**
|
|
443
|
+
* Render a document according to a RenderConfig.
|
|
444
|
+
*/
|
|
445
|
+
function renderDoc(context, config, hbs) {
|
|
446
|
+
const parts = [];
|
|
447
|
+
// Frontmatter
|
|
448
|
+
const fmObj = {};
|
|
449
|
+
for (const key of config.frontmatter) {
|
|
450
|
+
const v = get(context, key);
|
|
451
|
+
if (v !== undefined) {
|
|
452
|
+
fmObj[key] = v;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
if (Object.keys(fmObj).length > 0) {
|
|
456
|
+
parts.push('---');
|
|
457
|
+
parts.push(yaml.dump(fmObj, { skipInvalid: true }).trim());
|
|
458
|
+
parts.push('---');
|
|
459
|
+
parts.push('');
|
|
460
|
+
}
|
|
461
|
+
// Body
|
|
462
|
+
for (const section of config.body) {
|
|
463
|
+
const heading = renderSectionHeading(section, hbs, context);
|
|
464
|
+
parts.push(heading);
|
|
465
|
+
parts.push('');
|
|
466
|
+
const v = get(context, section.path);
|
|
467
|
+
const body = section.each
|
|
468
|
+
? renderEach(hbs, section, v)
|
|
469
|
+
: renderValueAsMarkdown(hbs, section, v);
|
|
470
|
+
if (body.trim()) {
|
|
471
|
+
parts.push(body.trim());
|
|
472
|
+
}
|
|
473
|
+
parts.push('');
|
|
474
|
+
}
|
|
475
|
+
return parts.join('\n').trim() + '\n';
|
|
476
|
+
}
|
|
477
|
+
|
|
337
478
|
/**
|
|
338
479
|
* @module util/normalizeError
|
|
339
480
|
*
|
|
@@ -820,15 +961,30 @@ async function applyRules(compiledRules, attributes, options = {}) {
|
|
|
820
961
|
log.warn(`JsonMap transformation failed: ${normalizeError(error).message}`);
|
|
821
962
|
}
|
|
822
963
|
}
|
|
964
|
+
// Build template context: attributes (with json spread at top) + map output
|
|
965
|
+
const context = {
|
|
966
|
+
...(attributes.json ?? {}),
|
|
967
|
+
...attributes,
|
|
968
|
+
...merged,
|
|
969
|
+
};
|
|
970
|
+
// Render via renderDoc if present
|
|
971
|
+
if (rule.render) {
|
|
972
|
+
try {
|
|
973
|
+
const result = renderDoc(context, rule.render, hbs);
|
|
974
|
+
if (result && result.trim()) {
|
|
975
|
+
renderedContent = result;
|
|
976
|
+
}
|
|
977
|
+
else {
|
|
978
|
+
log.warn(`renderDoc for rule "${rule.name}" rendered empty output. Falling back to raw content.`);
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
catch (error) {
|
|
982
|
+
log.warn(`renderDoc failed for rule "${rule.name}": ${normalizeError(error).message}. Falling back to raw content.`);
|
|
983
|
+
}
|
|
984
|
+
}
|
|
823
985
|
// Render template if present
|
|
824
986
|
if (rule.template && templateEngine) {
|
|
825
987
|
const templateKey = rule.name;
|
|
826
|
-
// Build template context: attributes (with json spread at top) + map output
|
|
827
|
-
const context = {
|
|
828
|
-
...(attributes.json ?? {}),
|
|
829
|
-
...attributes,
|
|
830
|
-
...merged,
|
|
831
|
-
};
|
|
832
988
|
try {
|
|
833
989
|
const result = templateEngine.render(templateKey, context);
|
|
834
990
|
if (result && result.trim()) {
|
|
@@ -894,6 +1050,26 @@ function buildAttributes(filePath, stats, extractedFrontmatter, extractedJson) {
|
|
|
894
1050
|
attrs.json = extractedJson;
|
|
895
1051
|
return attrs;
|
|
896
1052
|
}
|
|
1053
|
+
/**
|
|
1054
|
+
* Build synthetic file attributes from a path string (no actual file I/O).
|
|
1055
|
+
* Used by API handlers that need to match rules against paths without reading files.
|
|
1056
|
+
*
|
|
1057
|
+
* @param filePath - The file path.
|
|
1058
|
+
* @returns Synthetic file attributes with zeroed stats.
|
|
1059
|
+
*/
|
|
1060
|
+
function buildSyntheticAttributes(filePath) {
|
|
1061
|
+
const normalised = normalizeSlashes(filePath);
|
|
1062
|
+
return {
|
|
1063
|
+
file: {
|
|
1064
|
+
path: normalised,
|
|
1065
|
+
directory: normalizeSlashes(dirname(normalised)),
|
|
1066
|
+
filename: basename(normalised),
|
|
1067
|
+
extension: extname(normalised),
|
|
1068
|
+
sizeBytes: 0,
|
|
1069
|
+
modified: new Date(0).toISOString(),
|
|
1070
|
+
},
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
897
1073
|
|
|
898
1074
|
/**
|
|
899
1075
|
* @module rules/ajvSetup
|
|
@@ -961,8 +1137,14 @@ function compileRules(rules) {
|
|
|
961
1137
|
}
|
|
962
1138
|
|
|
963
1139
|
/**
|
|
964
|
-
* @module util/
|
|
965
|
-
*
|
|
1140
|
+
* @module util/sleep
|
|
1141
|
+
* Abort-aware async sleep.
|
|
1142
|
+
*/
|
|
1143
|
+
/**
|
|
1144
|
+
* Sleep for a given duration with optional abort support.
|
|
1145
|
+
*
|
|
1146
|
+
* @param ms - Duration in milliseconds.
|
|
1147
|
+
* @param signal - Optional abort signal.
|
|
966
1148
|
*/
|
|
967
1149
|
function sleep(ms, signal) {
|
|
968
1150
|
if (ms <= 0)
|
|
@@ -974,7 +1156,7 @@ function sleep(ms, signal) {
|
|
|
974
1156
|
}, ms);
|
|
975
1157
|
const onAbort = () => {
|
|
976
1158
|
cleanup();
|
|
977
|
-
reject(new Error('
|
|
1159
|
+
reject(new Error('Sleep aborted'));
|
|
978
1160
|
};
|
|
979
1161
|
const cleanup = () => {
|
|
980
1162
|
clearTimeout(timer);
|
|
@@ -990,6 +1172,11 @@ function sleep(ms, signal) {
|
|
|
990
1172
|
}
|
|
991
1173
|
});
|
|
992
1174
|
}
|
|
1175
|
+
|
|
1176
|
+
/**
|
|
1177
|
+
* @module util/retry
|
|
1178
|
+
* Small async retry helper with exponential backoff. Side effects: sleeps between attempts; can invoke onRetry callback for logging.
|
|
1179
|
+
*/
|
|
993
1180
|
function computeDelayMs(attempt, baseDelayMs, maxDelayMs, jitter = 0) {
|
|
994
1181
|
const exp = Math.max(0, attempt - 1);
|
|
995
1182
|
const raw = Math.min(maxDelayMs, baseDelayMs * 2 ** exp);
|
|
@@ -1283,7 +1470,12 @@ const apiConfigSchema = z.object({
|
|
|
1283
1470
|
.optional()
|
|
1284
1471
|
.describe('Host address for API server (e.g., "127.0.0.1", "0.0.0.0").'),
|
|
1285
1472
|
/** Port to listen on. */
|
|
1286
|
-
port: z.number().optional().describe('Port for API server (e.g.,
|
|
1473
|
+
port: z.number().optional().describe('Port for API server (e.g., 1936).'),
|
|
1474
|
+
/** Read endpoint cache TTL in milliseconds. */
|
|
1475
|
+
cacheTtlMs: z
|
|
1476
|
+
.number()
|
|
1477
|
+
.optional()
|
|
1478
|
+
.describe('TTL in milliseconds for caching read-heavy endpoints (e.g., /status, /config/query). Default: 30000.'),
|
|
1287
1479
|
});
|
|
1288
1480
|
/**
|
|
1289
1481
|
* Logging configuration.
|
|
@@ -1337,10 +1529,61 @@ const schemaReferenceSchema = z.union([
|
|
|
1337
1529
|
z.string().describe('Named reference to a global schema.'),
|
|
1338
1530
|
schemaObjectSchema,
|
|
1339
1531
|
]);
|
|
1532
|
+
/** Render body section. */
|
|
1533
|
+
const renderBodySectionSchema = z.object({
|
|
1534
|
+
/** Key path in the template context to render. */
|
|
1535
|
+
path: z.string().min(1).describe('Key path in template context to render.'),
|
|
1536
|
+
/** Markdown heading level for this section (1-6). */
|
|
1537
|
+
heading: z.number().min(1).max(6).describe('Markdown heading level (1-6).'),
|
|
1538
|
+
/** Override heading text (default: titlecased path). */
|
|
1539
|
+
label: z.string().optional().describe('Override heading text.'),
|
|
1540
|
+
/** Name of a registered Handlebars helper used as a format handler. */
|
|
1541
|
+
format: z
|
|
1542
|
+
.string()
|
|
1543
|
+
.optional()
|
|
1544
|
+
.describe('Name of a registered Handlebars helper used as a format handler.'),
|
|
1545
|
+
/** Additional args passed to the format helper. */
|
|
1546
|
+
formatArgs: z
|
|
1547
|
+
.array(z.unknown())
|
|
1548
|
+
.optional()
|
|
1549
|
+
.describe('Additional args passed to the format helper.'),
|
|
1550
|
+
/** If true, the value at path is treated as an array and iterated. */
|
|
1551
|
+
each: z
|
|
1552
|
+
.boolean()
|
|
1553
|
+
.optional()
|
|
1554
|
+
.describe('If true, the value at path is treated as an array and iterated.'),
|
|
1555
|
+
/** Handlebars template string for per-item heading text (used when each=true). */
|
|
1556
|
+
headingTemplate: z
|
|
1557
|
+
.string()
|
|
1558
|
+
.optional()
|
|
1559
|
+
.describe('Handlebars template string for per-item heading text (used when each=true).'),
|
|
1560
|
+
/** Key path within each item to use as renderable content (used when each=true). */
|
|
1561
|
+
contentPath: z
|
|
1562
|
+
.string()
|
|
1563
|
+
.optional()
|
|
1564
|
+
.describe('Key path within each item to use as renderable content (used when each=true).'),
|
|
1565
|
+
/** Key path within each item to sort by (used when each=true). */
|
|
1566
|
+
sort: z
|
|
1567
|
+
.string()
|
|
1568
|
+
.optional()
|
|
1569
|
+
.describe('Key path within each item to sort by (used when each=true).'),
|
|
1570
|
+
});
|
|
1571
|
+
/** Render config: YAML frontmatter + ordered body sections. */
|
|
1572
|
+
const renderConfigSchema = z.object({
|
|
1573
|
+
/** Keys to extract from context and include as YAML frontmatter. */
|
|
1574
|
+
frontmatter: z
|
|
1575
|
+
.array(z.string().min(1))
|
|
1576
|
+
.describe('Keys to extract from context and include as YAML frontmatter.'),
|
|
1577
|
+
/** Ordered markdown body sections. */
|
|
1578
|
+
body: z
|
|
1579
|
+
.array(renderBodySectionSchema)
|
|
1580
|
+
.describe('Ordered markdown body sections.'),
|
|
1581
|
+
});
|
|
1340
1582
|
/**
|
|
1341
1583
|
* An inference rule that enriches document metadata.
|
|
1342
1584
|
*/
|
|
1343
|
-
const inferenceRuleSchema = z
|
|
1585
|
+
const inferenceRuleSchema = z
|
|
1586
|
+
.object({
|
|
1344
1587
|
/** Unique name for this inference rule. */
|
|
1345
1588
|
name: z
|
|
1346
1589
|
.string()
|
|
@@ -1370,6 +1613,19 @@ const inferenceRuleSchema = z.object({
|
|
|
1370
1613
|
.string()
|
|
1371
1614
|
.optional()
|
|
1372
1615
|
.describe('Handlebars content template (inline string, named ref, or .hbs/.handlebars file path).'),
|
|
1616
|
+
/** Declarative structured renderer configuration (mutually exclusive with template). */
|
|
1617
|
+
render: renderConfigSchema
|
|
1618
|
+
.optional()
|
|
1619
|
+
.describe('Declarative render configuration for frontmatter + structured Markdown output (mutually exclusive with template).'),
|
|
1620
|
+
})
|
|
1621
|
+
.superRefine((val, ctx) => {
|
|
1622
|
+
if (val.render && val.template) {
|
|
1623
|
+
ctx.addIssue({
|
|
1624
|
+
code: 'custom',
|
|
1625
|
+
path: ['render'],
|
|
1626
|
+
message: 'render is mutually exclusive with template',
|
|
1627
|
+
});
|
|
1628
|
+
}
|
|
1373
1629
|
});
|
|
1374
1630
|
|
|
1375
1631
|
/**
|
|
@@ -1684,11 +1940,16 @@ function mergeAndValidateConfig(currentConfig, submittedPartial) {
|
|
|
1684
1940
|
function wrapHandler(fn, logger, label) {
|
|
1685
1941
|
return async (request, reply) => {
|
|
1686
1942
|
try {
|
|
1687
|
-
|
|
1943
|
+
const result = await fn(request, reply);
|
|
1944
|
+
if (!reply.sent) {
|
|
1945
|
+
void reply.send(result);
|
|
1946
|
+
}
|
|
1688
1947
|
}
|
|
1689
1948
|
catch (error) {
|
|
1690
1949
|
logger.error({ err: normalizeError(error) }, `${label} failed`);
|
|
1691
|
-
|
|
1950
|
+
if (!reply.sent) {
|
|
1951
|
+
void reply.status(500).send({ error: 'Internal server error' });
|
|
1952
|
+
}
|
|
1692
1953
|
}
|
|
1693
1954
|
};
|
|
1694
1955
|
}
|
|
@@ -1748,17 +2009,7 @@ function createConfigMatchHandler(options) {
|
|
|
1748
2009
|
return;
|
|
1749
2010
|
}
|
|
1750
2011
|
const matches = body.paths.map((path) => {
|
|
1751
|
-
const
|
|
1752
|
-
const attrs = {
|
|
1753
|
-
file: {
|
|
1754
|
-
path: normalised,
|
|
1755
|
-
directory: normalizeSlashes(dirname(normalised)),
|
|
1756
|
-
filename: basename(normalised),
|
|
1757
|
-
extension: extname(normalised),
|
|
1758
|
-
sizeBytes: 0,
|
|
1759
|
-
modified: new Date(0).toISOString(),
|
|
1760
|
-
},
|
|
1761
|
-
};
|
|
2012
|
+
const attrs = buildSyntheticAttributes(path);
|
|
1762
2013
|
// Find matching rules
|
|
1763
2014
|
const matchingRules = [];
|
|
1764
2015
|
for (const compiled of compiledRules) {
|
|
@@ -1767,6 +2018,7 @@ function createConfigMatchHandler(options) {
|
|
|
1767
2018
|
}
|
|
1768
2019
|
}
|
|
1769
2020
|
// Check watch scope: matches watch paths and not in ignored
|
|
2021
|
+
const normalised = attrs.file.path;
|
|
1770
2022
|
const watched = watchMatcher(normalised) && !ignoreMatcher?.(normalised);
|
|
1771
2023
|
return { rules: matchingRules, watched };
|
|
1772
2024
|
});
|
|
@@ -1918,6 +2170,9 @@ function resolveReferences(doc, resolveTypes) {
|
|
|
1918
2170
|
/**
|
|
1919
2171
|
* Create handler for POST /config/query.
|
|
1920
2172
|
*
|
|
2173
|
+
* Uses direct error handling (returns 400) rather than wrapHandler (which returns 500),
|
|
2174
|
+
* because invalid JSONPath expressions are client errors, not server errors.
|
|
2175
|
+
*
|
|
1921
2176
|
* @param deps - Route dependencies.
|
|
1922
2177
|
*/
|
|
1923
2178
|
function createConfigQueryHandler(deps) {
|
|
@@ -2033,7 +2288,7 @@ function createConfigSchemaHandler() {
|
|
|
2033
2288
|
/**
|
|
2034
2289
|
* Validate helper file references (mapHelpers, templateHelpers).
|
|
2035
2290
|
*/
|
|
2036
|
-
function validateHelperFiles(config) {
|
|
2291
|
+
function validateHelperFiles(config, configDir) {
|
|
2037
2292
|
const errors = [];
|
|
2038
2293
|
for (const section of ['mapHelpers', 'templateHelpers']) {
|
|
2039
2294
|
const helpers = config[section];
|
|
@@ -2042,15 +2297,18 @@ function validateHelperFiles(config) {
|
|
|
2042
2297
|
for (const [name, helper] of Object.entries(helpers)) {
|
|
2043
2298
|
if (!helper.path)
|
|
2044
2299
|
continue;
|
|
2045
|
-
|
|
2300
|
+
const resolvedPath = isAbsolute(helper.path)
|
|
2301
|
+
? helper.path
|
|
2302
|
+
: resolve(configDir, helper.path);
|
|
2303
|
+
if (!existsSync(resolvedPath)) {
|
|
2046
2304
|
errors.push({
|
|
2047
2305
|
path: `${section}.${name}.path`,
|
|
2048
|
-
message: `File not found: ${
|
|
2306
|
+
message: `File not found: ${resolvedPath}`,
|
|
2049
2307
|
});
|
|
2050
2308
|
continue;
|
|
2051
2309
|
}
|
|
2052
2310
|
try {
|
|
2053
|
-
readFileSync(
|
|
2311
|
+
readFileSync(resolvedPath, 'utf-8');
|
|
2054
2312
|
}
|
|
2055
2313
|
catch (err) {
|
|
2056
2314
|
errors.push({
|
|
@@ -2103,7 +2361,7 @@ function createConfigValidateHandler(deps) {
|
|
|
2103
2361
|
if (errors.length > 0) {
|
|
2104
2362
|
return { valid: false, errors };
|
|
2105
2363
|
}
|
|
2106
|
-
const helperErrors = validateHelperFiles(candidateRaw);
|
|
2364
|
+
const helperErrors = validateHelperFiles(candidateRaw, deps.configDir);
|
|
2107
2365
|
if (helperErrors.length > 0) {
|
|
2108
2366
|
return { valid: false, errors: helperErrors };
|
|
2109
2367
|
}
|
|
@@ -2207,17 +2465,7 @@ function getValueType(value) {
|
|
|
2207
2465
|
*/
|
|
2208
2466
|
function validateMetadataPayload(config, path, metadata) {
|
|
2209
2467
|
const compiled = compileRules(config.inferenceRules ?? []);
|
|
2210
|
-
const
|
|
2211
|
-
const attrs = {
|
|
2212
|
-
file: {
|
|
2213
|
-
path: normalised,
|
|
2214
|
-
directory: normalizeSlashes(dirname(normalised)),
|
|
2215
|
-
filename: basename(normalised),
|
|
2216
|
-
extension: extname(normalised),
|
|
2217
|
-
sizeBytes: 0,
|
|
2218
|
-
modified: new Date(0).toISOString(),
|
|
2219
|
-
},
|
|
2220
|
-
};
|
|
2468
|
+
const attrs = buildSyntheticAttributes(path);
|
|
2221
2469
|
const matched = compiled.filter((r) => r.validate(attrs));
|
|
2222
2470
|
const matchedNames = matched.map((m) => m.rule.name);
|
|
2223
2471
|
const schemaRefs = matched.flatMap((m) => m.rule.schema ?? []);
|
|
@@ -2491,7 +2739,6 @@ function createReindexHandler(deps) {
|
|
|
2491
2739
|
*/
|
|
2492
2740
|
function createRulesReapplyHandler(deps) {
|
|
2493
2741
|
return wrapHandler(async (request) => {
|
|
2494
|
-
await Promise.resolve();
|
|
2495
2742
|
const { globs } = request.body;
|
|
2496
2743
|
if (!Array.isArray(globs) || globs.length === 0) {
|
|
2497
2744
|
throw new Error('Missing required field: globs (non-empty string array)');
|
|
@@ -2534,7 +2781,6 @@ function createRulesReapplyHandler(deps) {
|
|
|
2534
2781
|
*/
|
|
2535
2782
|
function createRulesRegisterHandler(deps) {
|
|
2536
2783
|
return wrapHandler(async (request) => {
|
|
2537
|
-
await Promise.resolve();
|
|
2538
2784
|
const { source, rules } = request.body;
|
|
2539
2785
|
if (!source || typeof source !== 'string') {
|
|
2540
2786
|
throw new Error('Missing required field: source');
|
|
@@ -2545,11 +2791,11 @@ function createRulesRegisterHandler(deps) {
|
|
|
2545
2791
|
deps.virtualRuleStore.register(source, rules);
|
|
2546
2792
|
deps.onRulesChanged();
|
|
2547
2793
|
deps.logger.info({ source, ruleCount: rules.length }, 'Virtual rules registered');
|
|
2548
|
-
return {
|
|
2794
|
+
return await Promise.resolve({
|
|
2549
2795
|
source,
|
|
2550
2796
|
registered: rules.length,
|
|
2551
2797
|
totalVirtualRules: deps.virtualRuleStore.size,
|
|
2552
|
-
};
|
|
2798
|
+
});
|
|
2553
2799
|
}, deps.logger, 'RulesRegister');
|
|
2554
2800
|
}
|
|
2555
2801
|
|
|
@@ -2557,21 +2803,25 @@ function createRulesRegisterHandler(deps) {
|
|
|
2557
2803
|
* @module api/handlers/rulesUnregister
|
|
2558
2804
|
* Fastify route handler for DELETE /rules/unregister.
|
|
2559
2805
|
*/
|
|
2806
|
+
/**
|
|
2807
|
+
* Core unregister logic shared by body and param handlers.
|
|
2808
|
+
*/
|
|
2809
|
+
function unregisterSource(deps, source) {
|
|
2810
|
+
if (!source || typeof source !== 'string') {
|
|
2811
|
+
throw new Error('Missing required field: source');
|
|
2812
|
+
}
|
|
2813
|
+
const removed = deps.virtualRuleStore.unregister(source);
|
|
2814
|
+
if (removed)
|
|
2815
|
+
deps.onRulesChanged();
|
|
2816
|
+
deps.logger.info({ source, removed }, 'Virtual rules unregister');
|
|
2817
|
+
return { source, removed };
|
|
2818
|
+
}
|
|
2560
2819
|
/**
|
|
2561
2820
|
* Create handler for DELETE /rules/unregister (body-based).
|
|
2562
2821
|
*/
|
|
2563
2822
|
function createRulesUnregisterHandler(deps) {
|
|
2564
2823
|
return wrapHandler(async (request) => {
|
|
2565
|
-
|
|
2566
|
-
const { source } = request.body;
|
|
2567
|
-
if (!source || typeof source !== 'string') {
|
|
2568
|
-
throw new Error('Missing required field: source');
|
|
2569
|
-
}
|
|
2570
|
-
const removed = deps.virtualRuleStore.unregister(source);
|
|
2571
|
-
if (removed)
|
|
2572
|
-
deps.onRulesChanged();
|
|
2573
|
-
deps.logger.info({ source, removed }, 'Virtual rules unregister');
|
|
2574
|
-
return { source, removed };
|
|
2824
|
+
return Promise.resolve(unregisterSource(deps, request.body.source));
|
|
2575
2825
|
}, deps.logger, 'RulesUnregister');
|
|
2576
2826
|
}
|
|
2577
2827
|
/**
|
|
@@ -2579,16 +2829,7 @@ function createRulesUnregisterHandler(deps) {
|
|
|
2579
2829
|
*/
|
|
2580
2830
|
function createRulesUnregisterParamHandler(deps) {
|
|
2581
2831
|
return wrapHandler(async (request) => {
|
|
2582
|
-
|
|
2583
|
-
const { source } = request.params;
|
|
2584
|
-
if (!source || typeof source !== 'string') {
|
|
2585
|
-
throw new Error('Missing required param: source');
|
|
2586
|
-
}
|
|
2587
|
-
const removed = deps.virtualRuleStore.unregister(source);
|
|
2588
|
-
if (removed)
|
|
2589
|
-
deps.onRulesChanged();
|
|
2590
|
-
deps.logger.info({ source, removed }, 'Virtual rules unregister');
|
|
2591
|
-
return { source, removed };
|
|
2832
|
+
return Promise.resolve(unregisterSource(deps, request.params.source));
|
|
2592
2833
|
}, deps.logger, 'RulesUnregister');
|
|
2593
2834
|
}
|
|
2594
2835
|
|
|
@@ -2637,6 +2878,55 @@ function createStatusHandler(deps) {
|
|
|
2637
2878
|
};
|
|
2638
2879
|
}
|
|
2639
2880
|
|
|
2881
|
+
/**
|
|
2882
|
+
* In-memory response cache
|
|
2883
|
+
*/
|
|
2884
|
+
const cache = new Map();
|
|
2885
|
+
/**
|
|
2886
|
+
* Generates a deterministic hash for an object
|
|
2887
|
+
*/
|
|
2888
|
+
function hashObject(obj) {
|
|
2889
|
+
if (obj === undefined)
|
|
2890
|
+
return 'undefined';
|
|
2891
|
+
if (obj === null)
|
|
2892
|
+
return 'null';
|
|
2893
|
+
const str = typeof obj === 'string' ? obj : JSON.stringify(obj);
|
|
2894
|
+
return crypto.createHash('sha256').update(str).digest('hex');
|
|
2895
|
+
}
|
|
2896
|
+
/**
|
|
2897
|
+
* Higher-order function to wrap Fastify route handlers with an in-memory TTL cache.
|
|
2898
|
+
* Uses request method, URL, and body hash as the cache key.
|
|
2899
|
+
*
|
|
2900
|
+
* @param ttlMs - Time to live in milliseconds
|
|
2901
|
+
* @param handler - The original route handler
|
|
2902
|
+
* @returns A new route handler that implements caching
|
|
2903
|
+
*/
|
|
2904
|
+
function withCache(ttlMs, handler) {
|
|
2905
|
+
return async (req, reply) => {
|
|
2906
|
+
// Generate deterministic cache key: METHOD:URL:BODY_HASH
|
|
2907
|
+
const bodyHash = hashObject(req.body);
|
|
2908
|
+
const key = `${req.method}:${req.url}:${bodyHash}`;
|
|
2909
|
+
// Check cache
|
|
2910
|
+
const now = Date.now();
|
|
2911
|
+
const entry = cache.get(key);
|
|
2912
|
+
if (entry && entry.expiresAt > now) {
|
|
2913
|
+
return entry.value;
|
|
2914
|
+
}
|
|
2915
|
+
// Cache miss - call handler
|
|
2916
|
+
const result = await handler(req, reply);
|
|
2917
|
+
// Don't cache errors (Fastify reply properties might indicate error)
|
|
2918
|
+
if (reply.statusCode >= 400) {
|
|
2919
|
+
return result;
|
|
2920
|
+
}
|
|
2921
|
+
// Store in cache
|
|
2922
|
+
cache.set(key, {
|
|
2923
|
+
value: result,
|
|
2924
|
+
expiresAt: now + ttlMs,
|
|
2925
|
+
});
|
|
2926
|
+
return result;
|
|
2927
|
+
};
|
|
2928
|
+
}
|
|
2929
|
+
|
|
2640
2930
|
/**
|
|
2641
2931
|
* @module api/ReindexTracker
|
|
2642
2932
|
* Tracks reindex operation state for status reporting. Single instance shared across handlers.
|
|
@@ -2698,11 +2988,12 @@ function createApiServer(options) {
|
|
|
2698
2988
|
issuesManager,
|
|
2699
2989
|
}, scope);
|
|
2700
2990
|
};
|
|
2701
|
-
|
|
2991
|
+
const cacheTtlMs = config.api?.cacheTtlMs ?? 30000;
|
|
2992
|
+
app.get('/status', withCache(cacheTtlMs, createStatusHandler({
|
|
2702
2993
|
vectorStore,
|
|
2703
2994
|
collectionName: config.vectorStore.collectionName,
|
|
2704
2995
|
reindexTracker,
|
|
2705
|
-
}));
|
|
2996
|
+
})));
|
|
2706
2997
|
app.post('/metadata', createMetadataHandler({ processor, config, logger }));
|
|
2707
2998
|
const hybridConfig = config.search?.hybrid
|
|
2708
2999
|
? {
|
|
@@ -2723,10 +3014,10 @@ function createApiServer(options) {
|
|
|
2723
3014
|
logger,
|
|
2724
3015
|
}));
|
|
2725
3016
|
app.post('/config-reindex', createConfigReindexHandler({ config, processor, logger, reindexTracker }));
|
|
2726
|
-
app.get('/issues', createIssuesHandler({ issuesManager }));
|
|
2727
|
-
app.get('/config/schema', createConfigSchemaHandler());
|
|
3017
|
+
app.get('/issues', withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
|
|
3018
|
+
app.get('/config/schema', withCache(cacheTtlMs, createConfigSchemaHandler()));
|
|
2728
3019
|
app.post('/config/match', createConfigMatchHandler({ config, logger }));
|
|
2729
|
-
app.post('/config/query', createConfigQueryHandler({
|
|
3020
|
+
app.post('/config/query', withCache(cacheTtlMs, createConfigQueryHandler({
|
|
2730
3021
|
config,
|
|
2731
3022
|
valuesManager,
|
|
2732
3023
|
issuesManager,
|
|
@@ -2735,8 +3026,12 @@ function createApiServer(options) {
|
|
|
2735
3026
|
getVirtualRules: virtualRuleStore
|
|
2736
3027
|
? () => virtualRuleStore.getAll()
|
|
2737
3028
|
: undefined,
|
|
3029
|
+
})));
|
|
3030
|
+
app.post('/config/validate', createConfigValidateHandler({
|
|
3031
|
+
config,
|
|
3032
|
+
logger,
|
|
3033
|
+
configDir: dirname(configPath),
|
|
2738
3034
|
}));
|
|
2739
|
-
app.post('/config/validate', createConfigValidateHandler({ config, logger }));
|
|
2740
3035
|
app.post('/config/apply', createConfigApplyHandler({
|
|
2741
3036
|
config,
|
|
2742
3037
|
configPath,
|
|
@@ -3095,7 +3390,8 @@ const CONFIG_WATCH_DEFAULTS = {
|
|
|
3095
3390
|
/** Default API values. */
|
|
3096
3391
|
const API_DEFAULTS = {
|
|
3097
3392
|
host: '127.0.0.1',
|
|
3098
|
-
port:
|
|
3393
|
+
port: 1936,
|
|
3394
|
+
cacheTtlMs: 30000,
|
|
3099
3395
|
};
|
|
3100
3396
|
/** Default logging values. */
|
|
3101
3397
|
const LOGGING_DEFAULTS = {
|
|
@@ -3574,13 +3870,10 @@ const extractorRegistry = new Map([
|
|
|
3574
3870
|
* @returns Extracted text and optional structured data.
|
|
3575
3871
|
*/
|
|
3576
3872
|
async function extractText(filePath, extension, additionalExtractors) {
|
|
3577
|
-
//
|
|
3578
|
-
const registry =
|
|
3579
|
-
|
|
3580
|
-
|
|
3581
|
-
registry.set(ext, extractor);
|
|
3582
|
-
}
|
|
3583
|
-
}
|
|
3873
|
+
// Use base registry directly unless additional extractors provided
|
|
3874
|
+
const registry = additionalExtractors
|
|
3875
|
+
? new Map([...extractorRegistry, ...additionalExtractors])
|
|
3876
|
+
: extractorRegistry;
|
|
3584
3877
|
const extractor = registry.get(extension.toLowerCase());
|
|
3585
3878
|
if (extractor)
|
|
3586
3879
|
return extractor(filePath);
|
|
@@ -4666,28 +4959,7 @@ class SystemHealth {
|
|
|
4666
4959
|
if (delay <= 0)
|
|
4667
4960
|
return;
|
|
4668
4961
|
this.logger.warn({ delayMs: delay, consecutiveFailures: this.consecutiveFailures }, 'Backing off before next attempt');
|
|
4669
|
-
await
|
|
4670
|
-
const timer = setTimeout(() => {
|
|
4671
|
-
cleanup();
|
|
4672
|
-
resolve();
|
|
4673
|
-
}, delay);
|
|
4674
|
-
const onAbort = () => {
|
|
4675
|
-
cleanup();
|
|
4676
|
-
reject(new Error('Backoff aborted'));
|
|
4677
|
-
};
|
|
4678
|
-
const cleanup = () => {
|
|
4679
|
-
clearTimeout(timer);
|
|
4680
|
-
if (signal)
|
|
4681
|
-
signal.removeEventListener('abort', onAbort);
|
|
4682
|
-
};
|
|
4683
|
-
if (signal) {
|
|
4684
|
-
if (signal.aborted) {
|
|
4685
|
-
onAbort();
|
|
4686
|
-
return;
|
|
4687
|
-
}
|
|
4688
|
-
signal.addEventListener('abort', onAbort, { once: true });
|
|
4689
|
-
}
|
|
4690
|
-
});
|
|
4962
|
+
await sleep(delay, signal);
|
|
4691
4963
|
}
|
|
4692
4964
|
/** Current consecutive failure count. */
|
|
4693
4965
|
get failures() {
|
|
@@ -5515,7 +5787,7 @@ class JeevesWatcher {
|
|
|
5515
5787
|
});
|
|
5516
5788
|
await server.listen({
|
|
5517
5789
|
host: this.config.api?.host ?? '127.0.0.1',
|
|
5518
|
-
port: this.config.api?.port ??
|
|
5790
|
+
port: this.config.api?.port ?? 1936,
|
|
5519
5791
|
});
|
|
5520
5792
|
return server;
|
|
5521
5793
|
}
|