@karmaniverous/jeeves-watcher 0.6.9 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +243 -78
- package/dist/cli/jeeves-watcher/index.js +366 -96
- package/dist/index.d.ts +30 -0
- package/dist/index.js +367 -100
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { createRequire } from 'node:module';
|
|
3
3
|
import { Command } from '@commander-js/extra-typings';
|
|
4
4
|
import { readdir, stat, writeFile, rm, readFile, mkdir } from 'node:fs/promises';
|
|
5
|
-
import { resolve, dirname, join, extname, basename, relative } from 'node:path';
|
|
5
|
+
import { resolve, dirname, join, extname, basename, isAbsolute, relative } from 'node:path';
|
|
6
6
|
import picomatch from 'picomatch';
|
|
7
7
|
import { mkdirSync, existsSync, readFileSync, writeFileSync, statSync, readdirSync } from 'node:fs';
|
|
8
8
|
import { z, ZodError } from 'zod';
|
|
@@ -20,14 +20,15 @@ import { toMarkdown } from 'mdast-util-to-markdown';
|
|
|
20
20
|
import { capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
|
|
21
21
|
import rehypeParse from 'rehype-parse';
|
|
22
22
|
import { unified } from 'unified';
|
|
23
|
+
import yaml from 'js-yaml';
|
|
23
24
|
import { JSONPath } from 'jsonpath-plus';
|
|
24
25
|
import { createHash } from 'node:crypto';
|
|
26
|
+
import crypto from 'crypto';
|
|
25
27
|
import { cosmiconfig } from 'cosmiconfig';
|
|
26
28
|
import { GoogleGenerativeAIEmbeddings } from '@langchain/google-genai';
|
|
27
29
|
import pino from 'pino';
|
|
28
30
|
import { v5 } from 'uuid';
|
|
29
31
|
import * as cheerio from 'cheerio';
|
|
30
|
-
import yaml from 'js-yaml';
|
|
31
32
|
import mammoth from 'mammoth';
|
|
32
33
|
import { MarkdownTextSplitter, RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
|
|
33
34
|
import { QdrantClient } from '@qdrant/js-client-rest';
|
|
@@ -59,8 +60,14 @@ function normalizeError(error) {
|
|
|
59
60
|
}
|
|
60
61
|
|
|
61
62
|
/**
|
|
62
|
-
* @module util/
|
|
63
|
-
*
|
|
63
|
+
* @module util/sleep
|
|
64
|
+
* Abort-aware async sleep.
|
|
65
|
+
*/
|
|
66
|
+
/**
|
|
67
|
+
* Sleep for a given duration with optional abort support.
|
|
68
|
+
*
|
|
69
|
+
* @param ms - Duration in milliseconds.
|
|
70
|
+
* @param signal - Optional abort signal.
|
|
64
71
|
*/
|
|
65
72
|
function sleep(ms, signal) {
|
|
66
73
|
if (ms <= 0)
|
|
@@ -72,7 +79,7 @@ function sleep(ms, signal) {
|
|
|
72
79
|
}, ms);
|
|
73
80
|
const onAbort = () => {
|
|
74
81
|
cleanup();
|
|
75
|
-
reject(new Error('
|
|
82
|
+
reject(new Error('Sleep aborted'));
|
|
76
83
|
};
|
|
77
84
|
const cleanup = () => {
|
|
78
85
|
clearTimeout(timer);
|
|
@@ -88,6 +95,11 @@ function sleep(ms, signal) {
|
|
|
88
95
|
}
|
|
89
96
|
});
|
|
90
97
|
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* @module util/retry
|
|
101
|
+
* Small async retry helper with exponential backoff. Side effects: sleeps between attempts; can invoke onRetry callback for logging.
|
|
102
|
+
*/
|
|
91
103
|
function computeDelayMs(attempt, baseDelayMs, maxDelayMs, jitter = 0) {
|
|
92
104
|
const exp = Math.max(0, attempt - 1);
|
|
93
105
|
const raw = Math.min(maxDelayMs, baseDelayMs * 2 ** exp);
|
|
@@ -993,6 +1005,146 @@ async function buildTemplateEngine(rules, namedTemplates, templateHelpers, confi
|
|
|
993
1005
|
return engine;
|
|
994
1006
|
}
|
|
995
1007
|
|
|
1008
|
+
/**
|
|
1009
|
+
* @module templates/rebaseHeadings
|
|
1010
|
+
* Rebase Markdown heading levels so the minimum heading depth starts at baseHeading+1.
|
|
1011
|
+
*/
|
|
1012
|
+
/**
|
|
1013
|
+
* Rebase Markdown headings.
|
|
1014
|
+
*
|
|
1015
|
+
* Finds all ATX headings (lines starting with one or more '#'),
|
|
1016
|
+
* computes the minimum heading level present, and shifts all headings
|
|
1017
|
+
* so that the minimum becomes baseHeading+1.
|
|
1018
|
+
*/
|
|
1019
|
+
function rebaseHeadings(markdown, baseHeading) {
|
|
1020
|
+
if (!markdown.trim())
|
|
1021
|
+
return markdown;
|
|
1022
|
+
const lines = markdown.split(/\r?\n/);
|
|
1023
|
+
const headingLevels = [];
|
|
1024
|
+
for (const line of lines) {
|
|
1025
|
+
const m = line.match(/^(#{1,6})\s+/);
|
|
1026
|
+
if (m)
|
|
1027
|
+
headingLevels.push(m[1].length);
|
|
1028
|
+
}
|
|
1029
|
+
if (headingLevels.length === 0)
|
|
1030
|
+
return markdown;
|
|
1031
|
+
const minLevel = Math.min(...headingLevels);
|
|
1032
|
+
const targetMin = Math.min(6, Math.max(1, baseHeading + 1));
|
|
1033
|
+
const delta = targetMin - minLevel;
|
|
1034
|
+
if (delta === 0)
|
|
1035
|
+
return markdown;
|
|
1036
|
+
const rebased = lines.map((line) => {
|
|
1037
|
+
const m = line.match(/^(#{1,6})(\s+.*)$/);
|
|
1038
|
+
if (!m)
|
|
1039
|
+
return line;
|
|
1040
|
+
const level = m[1].length;
|
|
1041
|
+
const newLevel = Math.min(6, Math.max(1, level + delta));
|
|
1042
|
+
return `${'#'.repeat(newLevel)}${m[2]}`;
|
|
1043
|
+
});
|
|
1044
|
+
return rebased.join('\n');
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
/**
|
|
1048
|
+
* @module templates/renderDoc
|
|
1049
|
+
* Declarative renderer for YAML frontmatter + structured Markdown body.
|
|
1050
|
+
*/
|
|
1051
|
+
function renderSectionHeading(section, hbs, context = {}) {
|
|
1052
|
+
const level = Math.min(6, Math.max(1, section.heading));
|
|
1053
|
+
const label = section.label
|
|
1054
|
+
? hbs.compile(section.label)(context)
|
|
1055
|
+
: title(section.path);
|
|
1056
|
+
return `${'#'.repeat(level)} ${label}`;
|
|
1057
|
+
}
|
|
1058
|
+
function callFormatHelper(hbs, helperName, value, args) {
|
|
1059
|
+
const helper = hbs.helpers[helperName];
|
|
1060
|
+
if (typeof helper !== 'function') {
|
|
1061
|
+
throw new Error(`Format helper not found: ${helperName}`);
|
|
1062
|
+
}
|
|
1063
|
+
// Handlebars helpers may return SafeString or other values; coerce to string
|
|
1064
|
+
const safeArgs = args ?? [];
|
|
1065
|
+
const result = helper(value, ...safeArgs);
|
|
1066
|
+
return typeof result === 'string' ? result : String(result);
|
|
1067
|
+
}
|
|
1068
|
+
function renderValueAsMarkdown(hbs, section, value) {
|
|
1069
|
+
if (value === null || value === undefined)
|
|
1070
|
+
return '';
|
|
1071
|
+
if (section.format) {
|
|
1072
|
+
const md = callFormatHelper(hbs, section.format, value, section.formatArgs);
|
|
1073
|
+
return rebaseHeadings(md, section.heading);
|
|
1074
|
+
}
|
|
1075
|
+
const strValue = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
|
|
1076
|
+
return hbs.Utils.escapeExpression(strValue);
|
|
1077
|
+
}
|
|
1078
|
+
function renderEach(hbs, section, value) {
|
|
1079
|
+
if (!Array.isArray(value))
|
|
1080
|
+
return '';
|
|
1081
|
+
const items = [...value];
|
|
1082
|
+
if (section.sort) {
|
|
1083
|
+
items.sort((a, b) => {
|
|
1084
|
+
const av = get(a, section.sort);
|
|
1085
|
+
const bv = get(b, section.sort);
|
|
1086
|
+
const aStr = typeof av === 'string' ? av : JSON.stringify(av ?? '');
|
|
1087
|
+
const bStr = typeof bv === 'string' ? bv : JSON.stringify(bv ?? '');
|
|
1088
|
+
return aStr.localeCompare(bStr);
|
|
1089
|
+
});
|
|
1090
|
+
}
|
|
1091
|
+
const subHeadingLevel = Math.min(6, section.heading + 1);
|
|
1092
|
+
const headingTpl = section.headingTemplate
|
|
1093
|
+
? hbs.compile(section.headingTemplate)
|
|
1094
|
+
: undefined;
|
|
1095
|
+
const parts = [];
|
|
1096
|
+
for (const item of items) {
|
|
1097
|
+
const headingText = headingTpl
|
|
1098
|
+
? headingTpl(item)
|
|
1099
|
+
: '';
|
|
1100
|
+
if (headingText) {
|
|
1101
|
+
parts.push(`${'#'.repeat(subHeadingLevel)} ${headingText}`);
|
|
1102
|
+
}
|
|
1103
|
+
const contentVal = section.contentPath
|
|
1104
|
+
? get(item, section.contentPath)
|
|
1105
|
+
: item;
|
|
1106
|
+
const md = renderValueAsMarkdown(hbs, { ...section, heading: subHeadingLevel }, contentVal);
|
|
1107
|
+
if (md.trim())
|
|
1108
|
+
parts.push(md.trim());
|
|
1109
|
+
}
|
|
1110
|
+
return parts.join('\n\n');
|
|
1111
|
+
}
|
|
1112
|
+
/**
|
|
1113
|
+
* Render a document according to a RenderConfig.
|
|
1114
|
+
*/
|
|
1115
|
+
function renderDoc(context, config, hbs) {
|
|
1116
|
+
const parts = [];
|
|
1117
|
+
// Frontmatter
|
|
1118
|
+
const fmObj = {};
|
|
1119
|
+
for (const key of config.frontmatter) {
|
|
1120
|
+
const v = get(context, key);
|
|
1121
|
+
if (v !== undefined) {
|
|
1122
|
+
fmObj[key] = v;
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
if (Object.keys(fmObj).length > 0) {
|
|
1126
|
+
parts.push('---');
|
|
1127
|
+
parts.push(yaml.dump(fmObj, { skipInvalid: true }).trim());
|
|
1128
|
+
parts.push('---');
|
|
1129
|
+
parts.push('');
|
|
1130
|
+
}
|
|
1131
|
+
// Body
|
|
1132
|
+
for (const section of config.body) {
|
|
1133
|
+
const heading = renderSectionHeading(section, hbs, context);
|
|
1134
|
+
parts.push(heading);
|
|
1135
|
+
parts.push('');
|
|
1136
|
+
const v = get(context, section.path);
|
|
1137
|
+
const body = section.each
|
|
1138
|
+
? renderEach(hbs, section, v)
|
|
1139
|
+
: renderValueAsMarkdown(hbs, section, v);
|
|
1140
|
+
if (body.trim()) {
|
|
1141
|
+
parts.push(body.trim());
|
|
1142
|
+
}
|
|
1143
|
+
parts.push('');
|
|
1144
|
+
}
|
|
1145
|
+
return parts.join('\n').trim() + '\n';
|
|
1146
|
+
}
|
|
1147
|
+
|
|
996
1148
|
/**
|
|
997
1149
|
* @module rules/jsonMapLib
|
|
998
1150
|
* Creates the lib object for JsonMap transformations.
|
|
@@ -1454,15 +1606,30 @@ async function applyRules(compiledRules, attributes, options = {}) {
|
|
|
1454
1606
|
log.warn(`JsonMap transformation failed: ${normalizeError(error).message}`);
|
|
1455
1607
|
}
|
|
1456
1608
|
}
|
|
1609
|
+
// Build template context: attributes (with json spread at top) + map output
|
|
1610
|
+
const context = {
|
|
1611
|
+
...(attributes.json ?? {}),
|
|
1612
|
+
...attributes,
|
|
1613
|
+
...merged,
|
|
1614
|
+
};
|
|
1615
|
+
// Render via renderDoc if present
|
|
1616
|
+
if (rule.render) {
|
|
1617
|
+
try {
|
|
1618
|
+
const result = renderDoc(context, rule.render, hbs);
|
|
1619
|
+
if (result && result.trim()) {
|
|
1620
|
+
renderedContent = result;
|
|
1621
|
+
}
|
|
1622
|
+
else {
|
|
1623
|
+
log.warn(`renderDoc for rule "${rule.name}" rendered empty output. Falling back to raw content.`);
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
catch (error) {
|
|
1627
|
+
log.warn(`renderDoc failed for rule "${rule.name}": ${normalizeError(error).message}. Falling back to raw content.`);
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1457
1630
|
// Render template if present
|
|
1458
1631
|
if (rule.template && templateEngine) {
|
|
1459
1632
|
const templateKey = rule.name;
|
|
1460
|
-
// Build template context: attributes (with json spread at top) + map output
|
|
1461
|
-
const context = {
|
|
1462
|
-
...(attributes.json ?? {}),
|
|
1463
|
-
...attributes,
|
|
1464
|
-
...merged,
|
|
1465
|
-
};
|
|
1466
1633
|
try {
|
|
1467
1634
|
const result = templateEngine.render(templateKey, context);
|
|
1468
1635
|
if (result && result.trim()) {
|
|
@@ -1512,6 +1679,26 @@ function buildAttributes(filePath, stats, extractedFrontmatter, extractedJson) {
|
|
|
1512
1679
|
attrs.json = extractedJson;
|
|
1513
1680
|
return attrs;
|
|
1514
1681
|
}
|
|
1682
|
+
/**
|
|
1683
|
+
* Build synthetic file attributes from a path string (no actual file I/O).
|
|
1684
|
+
* Used by API handlers that need to match rules against paths without reading files.
|
|
1685
|
+
*
|
|
1686
|
+
* @param filePath - The file path.
|
|
1687
|
+
* @returns Synthetic file attributes with zeroed stats.
|
|
1688
|
+
*/
|
|
1689
|
+
function buildSyntheticAttributes(filePath) {
|
|
1690
|
+
const normalised = normalizeSlashes(filePath);
|
|
1691
|
+
return {
|
|
1692
|
+
file: {
|
|
1693
|
+
path: normalised,
|
|
1694
|
+
directory: normalizeSlashes(dirname(normalised)),
|
|
1695
|
+
filename: basename(normalised),
|
|
1696
|
+
extension: extname(normalised),
|
|
1697
|
+
sizeBytes: 0,
|
|
1698
|
+
modified: new Date(0).toISOString(),
|
|
1699
|
+
},
|
|
1700
|
+
};
|
|
1701
|
+
}
|
|
1515
1702
|
|
|
1516
1703
|
/**
|
|
1517
1704
|
* @module config/schemas/base
|
|
@@ -1592,7 +1779,12 @@ const apiConfigSchema = z.object({
|
|
|
1592
1779
|
.optional()
|
|
1593
1780
|
.describe('Host address for API server (e.g., "127.0.0.1", "0.0.0.0").'),
|
|
1594
1781
|
/** Port to listen on. */
|
|
1595
|
-
port: z.number().optional().describe('Port for API server (e.g.,
|
|
1782
|
+
port: z.number().optional().describe('Port for API server (e.g., 1936).'),
|
|
1783
|
+
/** Read endpoint cache TTL in milliseconds. */
|
|
1784
|
+
cacheTtlMs: z
|
|
1785
|
+
.number()
|
|
1786
|
+
.optional()
|
|
1787
|
+
.describe('TTL in milliseconds for caching read-heavy endpoints (e.g., /status, /config/query). Default: 30000.'),
|
|
1596
1788
|
});
|
|
1597
1789
|
/**
|
|
1598
1790
|
* Logging configuration.
|
|
@@ -1646,10 +1838,61 @@ const schemaReferenceSchema = z.union([
|
|
|
1646
1838
|
z.string().describe('Named reference to a global schema.'),
|
|
1647
1839
|
schemaObjectSchema,
|
|
1648
1840
|
]);
|
|
1841
|
+
/** Render body section. */
|
|
1842
|
+
const renderBodySectionSchema = z.object({
|
|
1843
|
+
/** Key path in the template context to render. */
|
|
1844
|
+
path: z.string().min(1).describe('Key path in template context to render.'),
|
|
1845
|
+
/** Markdown heading level for this section (1-6). */
|
|
1846
|
+
heading: z.number().min(1).max(6).describe('Markdown heading level (1-6).'),
|
|
1847
|
+
/** Override heading text (default: titlecased path). */
|
|
1848
|
+
label: z.string().optional().describe('Override heading text.'),
|
|
1849
|
+
/** Name of a registered Handlebars helper used as a format handler. */
|
|
1850
|
+
format: z
|
|
1851
|
+
.string()
|
|
1852
|
+
.optional()
|
|
1853
|
+
.describe('Name of a registered Handlebars helper used as a format handler.'),
|
|
1854
|
+
/** Additional args passed to the format helper. */
|
|
1855
|
+
formatArgs: z
|
|
1856
|
+
.array(z.unknown())
|
|
1857
|
+
.optional()
|
|
1858
|
+
.describe('Additional args passed to the format helper.'),
|
|
1859
|
+
/** If true, the value at path is treated as an array and iterated. */
|
|
1860
|
+
each: z
|
|
1861
|
+
.boolean()
|
|
1862
|
+
.optional()
|
|
1863
|
+
.describe('If true, the value at path is treated as an array and iterated.'),
|
|
1864
|
+
/** Handlebars template string for per-item heading text (used when each=true). */
|
|
1865
|
+
headingTemplate: z
|
|
1866
|
+
.string()
|
|
1867
|
+
.optional()
|
|
1868
|
+
.describe('Handlebars template string for per-item heading text (used when each=true).'),
|
|
1869
|
+
/** Key path within each item to use as renderable content (used when each=true). */
|
|
1870
|
+
contentPath: z
|
|
1871
|
+
.string()
|
|
1872
|
+
.optional()
|
|
1873
|
+
.describe('Key path within each item to use as renderable content (used when each=true).'),
|
|
1874
|
+
/** Key path within each item to sort by (used when each=true). */
|
|
1875
|
+
sort: z
|
|
1876
|
+
.string()
|
|
1877
|
+
.optional()
|
|
1878
|
+
.describe('Key path within each item to sort by (used when each=true).'),
|
|
1879
|
+
});
|
|
1880
|
+
/** Render config: YAML frontmatter + ordered body sections. */
|
|
1881
|
+
const renderConfigSchema = z.object({
|
|
1882
|
+
/** Keys to extract from context and include as YAML frontmatter. */
|
|
1883
|
+
frontmatter: z
|
|
1884
|
+
.array(z.string().min(1))
|
|
1885
|
+
.describe('Keys to extract from context and include as YAML frontmatter.'),
|
|
1886
|
+
/** Ordered markdown body sections. */
|
|
1887
|
+
body: z
|
|
1888
|
+
.array(renderBodySectionSchema)
|
|
1889
|
+
.describe('Ordered markdown body sections.'),
|
|
1890
|
+
});
|
|
1649
1891
|
/**
|
|
1650
1892
|
* An inference rule that enriches document metadata.
|
|
1651
1893
|
*/
|
|
1652
|
-
const inferenceRuleSchema = z
|
|
1894
|
+
const inferenceRuleSchema = z
|
|
1895
|
+
.object({
|
|
1653
1896
|
/** Unique name for this inference rule. */
|
|
1654
1897
|
name: z
|
|
1655
1898
|
.string()
|
|
@@ -1679,6 +1922,19 @@ const inferenceRuleSchema = z.object({
|
|
|
1679
1922
|
.string()
|
|
1680
1923
|
.optional()
|
|
1681
1924
|
.describe('Handlebars content template (inline string, named ref, or .hbs/.handlebars file path).'),
|
|
1925
|
+
/** Declarative structured renderer configuration (mutually exclusive with template). */
|
|
1926
|
+
render: renderConfigSchema
|
|
1927
|
+
.optional()
|
|
1928
|
+
.describe('Declarative render configuration for frontmatter + structured Markdown output (mutually exclusive with template).'),
|
|
1929
|
+
})
|
|
1930
|
+
.superRefine((val, ctx) => {
|
|
1931
|
+
if (val.render && val.template) {
|
|
1932
|
+
ctx.addIssue({
|
|
1933
|
+
code: 'custom',
|
|
1934
|
+
path: ['render'],
|
|
1935
|
+
message: 'render is mutually exclusive with template',
|
|
1936
|
+
});
|
|
1937
|
+
}
|
|
1682
1938
|
});
|
|
1683
1939
|
|
|
1684
1940
|
/**
|
|
@@ -2062,17 +2318,7 @@ function createConfigMatchHandler(options) {
|
|
|
2062
2318
|
return;
|
|
2063
2319
|
}
|
|
2064
2320
|
const matches = body.paths.map((path) => {
|
|
2065
|
-
const
|
|
2066
|
-
const attrs = {
|
|
2067
|
-
file: {
|
|
2068
|
-
path: normalised,
|
|
2069
|
-
directory: normalizeSlashes(dirname(normalised)),
|
|
2070
|
-
filename: basename(normalised),
|
|
2071
|
-
extension: extname(normalised),
|
|
2072
|
-
sizeBytes: 0,
|
|
2073
|
-
modified: new Date(0).toISOString(),
|
|
2074
|
-
},
|
|
2075
|
-
};
|
|
2321
|
+
const attrs = buildSyntheticAttributes(path);
|
|
2076
2322
|
// Find matching rules
|
|
2077
2323
|
const matchingRules = [];
|
|
2078
2324
|
for (const compiled of compiledRules) {
|
|
@@ -2081,6 +2327,7 @@ function createConfigMatchHandler(options) {
|
|
|
2081
2327
|
}
|
|
2082
2328
|
}
|
|
2083
2329
|
// Check watch scope: matches watch paths and not in ignored
|
|
2330
|
+
const normalised = attrs.file.path;
|
|
2084
2331
|
const watched = watchMatcher(normalised) && !ignoreMatcher?.(normalised);
|
|
2085
2332
|
return { rules: matchingRules, watched };
|
|
2086
2333
|
});
|
|
@@ -2232,6 +2479,9 @@ function resolveReferences(doc, resolveTypes) {
|
|
|
2232
2479
|
/**
|
|
2233
2480
|
* Create handler for POST /config/query.
|
|
2234
2481
|
*
|
|
2482
|
+
* Uses direct error handling (returns 400) rather than wrapHandler (which returns 500),
|
|
2483
|
+
* because invalid JSONPath expressions are client errors, not server errors.
|
|
2484
|
+
*
|
|
2235
2485
|
* @param deps - Route dependencies.
|
|
2236
2486
|
*/
|
|
2237
2487
|
function createConfigQueryHandler(deps) {
|
|
@@ -2347,7 +2597,7 @@ function createConfigSchemaHandler() {
|
|
|
2347
2597
|
/**
|
|
2348
2598
|
* Validate helper file references (mapHelpers, templateHelpers).
|
|
2349
2599
|
*/
|
|
2350
|
-
function validateHelperFiles(config) {
|
|
2600
|
+
function validateHelperFiles(config, configDir) {
|
|
2351
2601
|
const errors = [];
|
|
2352
2602
|
for (const section of ['mapHelpers', 'templateHelpers']) {
|
|
2353
2603
|
const helpers = config[section];
|
|
@@ -2356,15 +2606,18 @@ function validateHelperFiles(config) {
|
|
|
2356
2606
|
for (const [name, helper] of Object.entries(helpers)) {
|
|
2357
2607
|
if (!helper.path)
|
|
2358
2608
|
continue;
|
|
2359
|
-
|
|
2609
|
+
const resolvedPath = isAbsolute(helper.path)
|
|
2610
|
+
? helper.path
|
|
2611
|
+
: resolve(configDir, helper.path);
|
|
2612
|
+
if (!existsSync(resolvedPath)) {
|
|
2360
2613
|
errors.push({
|
|
2361
2614
|
path: `${section}.${name}.path`,
|
|
2362
|
-
message: `File not found: ${
|
|
2615
|
+
message: `File not found: ${resolvedPath}`,
|
|
2363
2616
|
});
|
|
2364
2617
|
continue;
|
|
2365
2618
|
}
|
|
2366
2619
|
try {
|
|
2367
|
-
readFileSync(
|
|
2620
|
+
readFileSync(resolvedPath, 'utf-8');
|
|
2368
2621
|
}
|
|
2369
2622
|
catch (err) {
|
|
2370
2623
|
errors.push({
|
|
@@ -2417,7 +2670,7 @@ function createConfigValidateHandler(deps) {
|
|
|
2417
2670
|
if (errors.length > 0) {
|
|
2418
2671
|
return { valid: false, errors };
|
|
2419
2672
|
}
|
|
2420
|
-
const helperErrors = validateHelperFiles(candidateRaw);
|
|
2673
|
+
const helperErrors = validateHelperFiles(candidateRaw, deps.configDir);
|
|
2421
2674
|
if (helperErrors.length > 0) {
|
|
2422
2675
|
return { valid: false, errors: helperErrors };
|
|
2423
2676
|
}
|
|
@@ -2521,17 +2774,7 @@ function getValueType(value) {
|
|
|
2521
2774
|
*/
|
|
2522
2775
|
function validateMetadataPayload(config, path, metadata) {
|
|
2523
2776
|
const compiled = compileRules(config.inferenceRules ?? []);
|
|
2524
|
-
const
|
|
2525
|
-
const attrs = {
|
|
2526
|
-
file: {
|
|
2527
|
-
path: normalised,
|
|
2528
|
-
directory: normalizeSlashes(dirname(normalised)),
|
|
2529
|
-
filename: basename(normalised),
|
|
2530
|
-
extension: extname(normalised),
|
|
2531
|
-
sizeBytes: 0,
|
|
2532
|
-
modified: new Date(0).toISOString(),
|
|
2533
|
-
},
|
|
2534
|
-
};
|
|
2777
|
+
const attrs = buildSyntheticAttributes(path);
|
|
2535
2778
|
const matched = compiled.filter((r) => r.validate(attrs));
|
|
2536
2779
|
const matchedNames = matched.map((m) => m.rule.name);
|
|
2537
2780
|
const schemaRefs = matched.flatMap((m) => m.rule.schema ?? []);
|
|
@@ -2805,7 +3048,6 @@ function createReindexHandler(deps) {
|
|
|
2805
3048
|
*/
|
|
2806
3049
|
function createRulesReapplyHandler(deps) {
|
|
2807
3050
|
return wrapHandler(async (request) => {
|
|
2808
|
-
await Promise.resolve();
|
|
2809
3051
|
const { globs } = request.body;
|
|
2810
3052
|
if (!Array.isArray(globs) || globs.length === 0) {
|
|
2811
3053
|
throw new Error('Missing required field: globs (non-empty string array)');
|
|
@@ -2848,7 +3090,6 @@ function createRulesReapplyHandler(deps) {
|
|
|
2848
3090
|
*/
|
|
2849
3091
|
function createRulesRegisterHandler(deps) {
|
|
2850
3092
|
return wrapHandler(async (request) => {
|
|
2851
|
-
await Promise.resolve();
|
|
2852
3093
|
const { source, rules } = request.body;
|
|
2853
3094
|
if (!source || typeof source !== 'string') {
|
|
2854
3095
|
throw new Error('Missing required field: source');
|
|
@@ -2859,11 +3100,11 @@ function createRulesRegisterHandler(deps) {
|
|
|
2859
3100
|
deps.virtualRuleStore.register(source, rules);
|
|
2860
3101
|
deps.onRulesChanged();
|
|
2861
3102
|
deps.logger.info({ source, ruleCount: rules.length }, 'Virtual rules registered');
|
|
2862
|
-
return {
|
|
3103
|
+
return await Promise.resolve({
|
|
2863
3104
|
source,
|
|
2864
3105
|
registered: rules.length,
|
|
2865
3106
|
totalVirtualRules: deps.virtualRuleStore.size,
|
|
2866
|
-
};
|
|
3107
|
+
});
|
|
2867
3108
|
}, deps.logger, 'RulesRegister');
|
|
2868
3109
|
}
|
|
2869
3110
|
|
|
@@ -2871,21 +3112,25 @@ function createRulesRegisterHandler(deps) {
|
|
|
2871
3112
|
* @module api/handlers/rulesUnregister
|
|
2872
3113
|
* Fastify route handler for DELETE /rules/unregister.
|
|
2873
3114
|
*/
|
|
3115
|
+
/**
|
|
3116
|
+
* Core unregister logic shared by body and param handlers.
|
|
3117
|
+
*/
|
|
3118
|
+
function unregisterSource(deps, source) {
|
|
3119
|
+
if (!source || typeof source !== 'string') {
|
|
3120
|
+
throw new Error('Missing required field: source');
|
|
3121
|
+
}
|
|
3122
|
+
const removed = deps.virtualRuleStore.unregister(source);
|
|
3123
|
+
if (removed)
|
|
3124
|
+
deps.onRulesChanged();
|
|
3125
|
+
deps.logger.info({ source, removed }, 'Virtual rules unregister');
|
|
3126
|
+
return { source, removed };
|
|
3127
|
+
}
|
|
2874
3128
|
/**
|
|
2875
3129
|
* Create handler for DELETE /rules/unregister (body-based).
|
|
2876
3130
|
*/
|
|
2877
3131
|
function createRulesUnregisterHandler(deps) {
|
|
2878
3132
|
return wrapHandler(async (request) => {
|
|
2879
|
-
|
|
2880
|
-
const { source } = request.body;
|
|
2881
|
-
if (!source || typeof source !== 'string') {
|
|
2882
|
-
throw new Error('Missing required field: source');
|
|
2883
|
-
}
|
|
2884
|
-
const removed = deps.virtualRuleStore.unregister(source);
|
|
2885
|
-
if (removed)
|
|
2886
|
-
deps.onRulesChanged();
|
|
2887
|
-
deps.logger.info({ source, removed }, 'Virtual rules unregister');
|
|
2888
|
-
return { source, removed };
|
|
3133
|
+
return Promise.resolve(unregisterSource(deps, request.body.source));
|
|
2889
3134
|
}, deps.logger, 'RulesUnregister');
|
|
2890
3135
|
}
|
|
2891
3136
|
/**
|
|
@@ -2893,16 +3138,7 @@ function createRulesUnregisterHandler(deps) {
|
|
|
2893
3138
|
*/
|
|
2894
3139
|
function createRulesUnregisterParamHandler(deps) {
|
|
2895
3140
|
return wrapHandler(async (request) => {
|
|
2896
|
-
|
|
2897
|
-
const { source } = request.params;
|
|
2898
|
-
if (!source || typeof source !== 'string') {
|
|
2899
|
-
throw new Error('Missing required param: source');
|
|
2900
|
-
}
|
|
2901
|
-
const removed = deps.virtualRuleStore.unregister(source);
|
|
2902
|
-
if (removed)
|
|
2903
|
-
deps.onRulesChanged();
|
|
2904
|
-
deps.logger.info({ source, removed }, 'Virtual rules unregister');
|
|
2905
|
-
return { source, removed };
|
|
3141
|
+
return Promise.resolve(unregisterSource(deps, request.params.source));
|
|
2906
3142
|
}, deps.logger, 'RulesUnregister');
|
|
2907
3143
|
}
|
|
2908
3144
|
|
|
@@ -2951,6 +3187,55 @@ function createStatusHandler(deps) {
|
|
|
2951
3187
|
};
|
|
2952
3188
|
}
|
|
2953
3189
|
|
|
3190
|
+
/**
|
|
3191
|
+
* In-memory response cache
|
|
3192
|
+
*/
|
|
3193
|
+
const cache = new Map();
|
|
3194
|
+
/**
|
|
3195
|
+
* Generates a deterministic hash for an object
|
|
3196
|
+
*/
|
|
3197
|
+
function hashObject(obj) {
|
|
3198
|
+
if (obj === undefined)
|
|
3199
|
+
return 'undefined';
|
|
3200
|
+
if (obj === null)
|
|
3201
|
+
return 'null';
|
|
3202
|
+
const str = typeof obj === 'string' ? obj : JSON.stringify(obj);
|
|
3203
|
+
return crypto.createHash('sha256').update(str).digest('hex');
|
|
3204
|
+
}
|
|
3205
|
+
/**
|
|
3206
|
+
* Higher-order function to wrap Fastify route handlers with an in-memory TTL cache.
|
|
3207
|
+
* Uses request method, URL, and body hash as the cache key.
|
|
3208
|
+
*
|
|
3209
|
+
* @param ttlMs - Time to live in milliseconds
|
|
3210
|
+
* @param handler - The original route handler
|
|
3211
|
+
* @returns A new route handler that implements caching
|
|
3212
|
+
*/
|
|
3213
|
+
function withCache(ttlMs, handler) {
|
|
3214
|
+
return async (req, reply) => {
|
|
3215
|
+
// Generate deterministic cache key: METHOD:URL:BODY_HASH
|
|
3216
|
+
const bodyHash = hashObject(req.body);
|
|
3217
|
+
const key = `${req.method}:${req.url}:${bodyHash}`;
|
|
3218
|
+
// Check cache
|
|
3219
|
+
const now = Date.now();
|
|
3220
|
+
const entry = cache.get(key);
|
|
3221
|
+
if (entry && entry.expiresAt > now) {
|
|
3222
|
+
return entry.value;
|
|
3223
|
+
}
|
|
3224
|
+
// Cache miss - call handler
|
|
3225
|
+
const result = await handler(req, reply);
|
|
3226
|
+
// Don't cache errors (Fastify reply properties might indicate error)
|
|
3227
|
+
if (reply.statusCode >= 400) {
|
|
3228
|
+
return result;
|
|
3229
|
+
}
|
|
3230
|
+
// Store in cache
|
|
3231
|
+
cache.set(key, {
|
|
3232
|
+
value: result,
|
|
3233
|
+
expiresAt: now + ttlMs,
|
|
3234
|
+
});
|
|
3235
|
+
return result;
|
|
3236
|
+
};
|
|
3237
|
+
}
|
|
3238
|
+
|
|
2954
3239
|
/**
|
|
2955
3240
|
* @module api/ReindexTracker
|
|
2956
3241
|
* Tracks reindex operation state for status reporting. Single instance shared across handlers.
|
|
@@ -3012,11 +3297,12 @@ function createApiServer(options) {
|
|
|
3012
3297
|
issuesManager,
|
|
3013
3298
|
}, scope);
|
|
3014
3299
|
};
|
|
3015
|
-
|
|
3300
|
+
const cacheTtlMs = config.api?.cacheTtlMs ?? 30000;
|
|
3301
|
+
app.get('/status', withCache(cacheTtlMs, createStatusHandler({
|
|
3016
3302
|
vectorStore,
|
|
3017
3303
|
collectionName: config.vectorStore.collectionName,
|
|
3018
3304
|
reindexTracker,
|
|
3019
|
-
}));
|
|
3305
|
+
})));
|
|
3020
3306
|
app.post('/metadata', createMetadataHandler({ processor, config, logger }));
|
|
3021
3307
|
const hybridConfig = config.search?.hybrid
|
|
3022
3308
|
? {
|
|
@@ -3037,10 +3323,10 @@ function createApiServer(options) {
|
|
|
3037
3323
|
logger,
|
|
3038
3324
|
}));
|
|
3039
3325
|
app.post('/config-reindex', createConfigReindexHandler({ config, processor, logger, reindexTracker }));
|
|
3040
|
-
app.get('/issues', createIssuesHandler({ issuesManager }));
|
|
3041
|
-
app.get('/config/schema', createConfigSchemaHandler());
|
|
3326
|
+
app.get('/issues', withCache(cacheTtlMs, createIssuesHandler({ issuesManager })));
|
|
3327
|
+
app.get('/config/schema', withCache(cacheTtlMs, createConfigSchemaHandler()));
|
|
3042
3328
|
app.post('/config/match', createConfigMatchHandler({ config, logger }));
|
|
3043
|
-
app.post('/config/query', createConfigQueryHandler({
|
|
3329
|
+
app.post('/config/query', withCache(cacheTtlMs, createConfigQueryHandler({
|
|
3044
3330
|
config,
|
|
3045
3331
|
valuesManager,
|
|
3046
3332
|
issuesManager,
|
|
@@ -3049,8 +3335,12 @@ function createApiServer(options) {
|
|
|
3049
3335
|
getVirtualRules: virtualRuleStore
|
|
3050
3336
|
? () => virtualRuleStore.getAll()
|
|
3051
3337
|
: undefined,
|
|
3338
|
+
})));
|
|
3339
|
+
app.post('/config/validate', createConfigValidateHandler({
|
|
3340
|
+
config,
|
|
3341
|
+
logger,
|
|
3342
|
+
configDir: dirname(configPath),
|
|
3052
3343
|
}));
|
|
3053
|
-
app.post('/config/validate', createConfigValidateHandler({ config, logger }));
|
|
3054
3344
|
app.post('/config/apply', createConfigApplyHandler({
|
|
3055
3345
|
config,
|
|
3056
3346
|
configPath,
|
|
@@ -3104,6 +3394,7 @@ const CONFIG_WATCH_DEFAULTS = {
|
|
|
3104
3394
|
const API_DEFAULTS = {
|
|
3105
3395
|
host: '127.0.0.1',
|
|
3106
3396
|
port: 1936,
|
|
3397
|
+
cacheTtlMs: 30000,
|
|
3107
3398
|
};
|
|
3108
3399
|
/** Default logging values. */
|
|
3109
3400
|
const LOGGING_DEFAULTS = {
|
|
@@ -3603,8 +3894,8 @@ const extractorRegistry = new Map([
|
|
|
3603
3894
|
* @returns Extracted text and optional structured data.
|
|
3604
3895
|
*/
|
|
3605
3896
|
async function extractText(filePath, extension, additionalExtractors) {
|
|
3606
|
-
//
|
|
3607
|
-
const registry =
|
|
3897
|
+
// Use base registry directly unless additional extractors provided
|
|
3898
|
+
const registry = extractorRegistry;
|
|
3608
3899
|
const extractor = registry.get(extension.toLowerCase());
|
|
3609
3900
|
if (extractor)
|
|
3610
3901
|
return extractor(filePath);
|
|
@@ -4690,28 +4981,7 @@ class SystemHealth {
|
|
|
4690
4981
|
if (delay <= 0)
|
|
4691
4982
|
return;
|
|
4692
4983
|
this.logger.warn({ delayMs: delay, consecutiveFailures: this.consecutiveFailures }, 'Backing off before next attempt');
|
|
4693
|
-
await
|
|
4694
|
-
const timer = setTimeout(() => {
|
|
4695
|
-
cleanup();
|
|
4696
|
-
resolve();
|
|
4697
|
-
}, delay);
|
|
4698
|
-
const onAbort = () => {
|
|
4699
|
-
cleanup();
|
|
4700
|
-
reject(new Error('Backoff aborted'));
|
|
4701
|
-
};
|
|
4702
|
-
const cleanup = () => {
|
|
4703
|
-
clearTimeout(timer);
|
|
4704
|
-
if (signal)
|
|
4705
|
-
signal.removeEventListener('abort', onAbort);
|
|
4706
|
-
};
|
|
4707
|
-
if (signal) {
|
|
4708
|
-
if (signal.aborted) {
|
|
4709
|
-
onAbort();
|
|
4710
|
-
return;
|
|
4711
|
-
}
|
|
4712
|
-
signal.addEventListener('abort', onAbort, { once: true });
|
|
4713
|
-
}
|
|
4714
|
-
});
|
|
4984
|
+
await sleep(delay, signal);
|
|
4715
4985
|
}
|
|
4716
4986
|
/** Current consecutive failure count. */
|
|
4717
4987
|
get failures() {
|
|
@@ -5539,7 +5809,7 @@ class JeevesWatcher {
|
|
|
5539
5809
|
});
|
|
5540
5810
|
await server.listen({
|
|
5541
5811
|
host: this.config.api?.host ?? '127.0.0.1',
|
|
5542
|
-
port: this.config.api?.port ??
|
|
5812
|
+
port: this.config.api?.port ?? 1936,
|
|
5543
5813
|
});
|
|
5544
5814
|
return server;
|
|
5545
5815
|
}
|
|
@@ -6107,7 +6377,7 @@ cli
|
|
|
6107
6377
|
console.log(` Watch paths: ${config.watch.paths.join(', ')}`);
|
|
6108
6378
|
console.log(` Embedding: ${config.embedding.provider}/${config.embedding.model}`);
|
|
6109
6379
|
console.log(` Vector store: ${config.vectorStore.url} (${config.vectorStore.collectionName})`);
|
|
6110
|
-
console.log(` API: ${config.api?.host ?? '127.0.0.1'}:${String(config.api?.port ??
|
|
6380
|
+
console.log(` API: ${config.api?.host ?? '127.0.0.1'}:${String(config.api?.port ?? 1936)}`);
|
|
6111
6381
|
}
|
|
6112
6382
|
catch (error) {
|
|
6113
6383
|
console.error('Config invalid:', error);
|