@karmaniverous/jeeves-watcher 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +38 -11
- package/dist/cjs/index.js +278 -168
- package/dist/cli/jeeves-watcher/index.js +279 -169
- package/dist/index.d.ts +10 -9
- package/dist/index.iife.js +278 -169
- package/dist/index.iife.min.js +1 -1
- package/dist/mjs/index.js +279 -169
- package/package.json +1 -1
|
@@ -3,12 +3,14 @@ import { Command } from '@commander-js/extra-typings';
|
|
|
3
3
|
import { dirname, resolve, relative, join, extname, basename } from 'node:path';
|
|
4
4
|
import { existsSync, statSync, readdirSync, readFileSync } from 'node:fs';
|
|
5
5
|
import ignore from 'ignore';
|
|
6
|
+
import { pathToFileURL } from 'node:url';
|
|
7
|
+
import { JsonMap, jsonMapMapSchema } from '@karmaniverous/jsonmap';
|
|
8
|
+
import { get, capitalize, title, camel, snake, dash, isEqual, omit } from 'radash';
|
|
6
9
|
import Handlebars from 'handlebars';
|
|
7
10
|
import dayjs from 'dayjs';
|
|
8
11
|
import { toMdast } from 'hast-util-to-mdast';
|
|
9
12
|
import { fromADF } from 'mdast-util-from-adf';
|
|
10
13
|
import { toMarkdown } from 'mdast-util-to-markdown';
|
|
11
|
-
import { capitalize, title, camel, snake, dash, isEqual, omit, get } from 'radash';
|
|
12
14
|
import rehypeParse from 'rehype-parse';
|
|
13
15
|
import { unified } from 'unified';
|
|
14
16
|
import chokidar from 'chokidar';
|
|
@@ -18,7 +20,6 @@ import picomatch from 'picomatch';
|
|
|
18
20
|
import { createHash } from 'node:crypto';
|
|
19
21
|
import { cosmiconfig } from 'cosmiconfig';
|
|
20
22
|
import { z, ZodError } from 'zod';
|
|
21
|
-
import { jsonMapMapSchema, JsonMap } from '@karmaniverous/jsonmap';
|
|
22
23
|
import { GoogleGenerativeAIEmbeddings } from '@langchain/google-genai';
|
|
23
24
|
import pino from 'pino';
|
|
24
25
|
import { v5 } from 'uuid';
|
|
@@ -242,6 +243,251 @@ class GitignoreFilter {
|
|
|
242
243
|
}
|
|
243
244
|
}
|
|
244
245
|
|
|
246
|
+
/**
|
|
247
|
+
* @module rules/templates
|
|
248
|
+
* Resolves template variables (`${path.to.value}`) in rule `set` objects against file attributes.
|
|
249
|
+
*/
|
|
250
|
+
/**
|
|
251
|
+
* Resolve `${template.vars}` in a value against the given attributes.
|
|
252
|
+
*
|
|
253
|
+
* @param value - The value to resolve.
|
|
254
|
+
* @param attributes - The file attributes for variable lookup.
|
|
255
|
+
* @returns The resolved value.
|
|
256
|
+
*/
|
|
257
|
+
function resolveTemplateVars(value, attributes) {
|
|
258
|
+
if (typeof value !== 'string')
|
|
259
|
+
return value;
|
|
260
|
+
return value.replace(/\$\{([^}]+)\}/g, (_match, varPath) => {
|
|
261
|
+
const current = get(attributes, varPath);
|
|
262
|
+
if (current === null || current === undefined)
|
|
263
|
+
return '';
|
|
264
|
+
return typeof current === 'string' ? current : JSON.stringify(current);
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Resolve all template variables in a `set` object.
|
|
269
|
+
*
|
|
270
|
+
* @param setObj - The key-value pairs to resolve.
|
|
271
|
+
* @param attributes - The file attributes for variable lookup.
|
|
272
|
+
* @returns The resolved key-value pairs.
|
|
273
|
+
*/
|
|
274
|
+
function resolveSet(setObj, attributes) {
|
|
275
|
+
const result = {};
|
|
276
|
+
for (const [key, value] of Object.entries(setObj)) {
|
|
277
|
+
result[key] = resolveTemplateVars(value, attributes);
|
|
278
|
+
}
|
|
279
|
+
return result;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* @module rules/apply
|
|
284
|
+
* Applies compiled inference rules to file attributes, producing merged metadata via template resolution and JsonMap transforms.
|
|
285
|
+
*/
|
|
286
|
+
/**
|
|
287
|
+
* Create the lib object for JsonMap transformations.
|
|
288
|
+
*
|
|
289
|
+
* @param configDir - Optional config directory for resolving relative file paths in lookups.
|
|
290
|
+
* @returns The lib object.
|
|
291
|
+
*/
|
|
292
|
+
function createJsonMapLib(configDir, customLib) {
|
|
293
|
+
// Cache loaded JSON files within a single applyRules invocation.
|
|
294
|
+
const jsonCache = new Map();
|
|
295
|
+
const loadJson = (filePath) => {
|
|
296
|
+
const resolvedPath = configDir ? resolve(configDir, filePath) : filePath;
|
|
297
|
+
if (!jsonCache.has(resolvedPath)) {
|
|
298
|
+
const raw = readFileSync(resolvedPath, 'utf-8');
|
|
299
|
+
jsonCache.set(resolvedPath, JSON.parse(raw));
|
|
300
|
+
}
|
|
301
|
+
return jsonCache.get(resolvedPath);
|
|
302
|
+
};
|
|
303
|
+
return {
|
|
304
|
+
split: (str, separator) => str.split(separator),
|
|
305
|
+
slice: (arr, start, end) => arr.slice(start, end),
|
|
306
|
+
join: (arr, separator) => arr.join(separator),
|
|
307
|
+
toLowerCase: (str) => str.toLowerCase(),
|
|
308
|
+
replace: (str, search, replacement) => str.replace(search, replacement),
|
|
309
|
+
get: (obj, path) => get(obj, path),
|
|
310
|
+
/**
|
|
311
|
+
* Load a JSON file (relative to configDir) and look up a value by key,
|
|
312
|
+
* optionally drilling into a sub-path.
|
|
313
|
+
*
|
|
314
|
+
* @param filePath - Path to a JSON file (resolved relative to configDir).
|
|
315
|
+
* @param key - Top-level key to look up.
|
|
316
|
+
* @param field - Optional dot-path into the looked-up entry.
|
|
317
|
+
* @returns The resolved value, or null if not found.
|
|
318
|
+
*/
|
|
319
|
+
lookupJson: (filePath, key, field) => {
|
|
320
|
+
const data = loadJson(filePath);
|
|
321
|
+
const entry = data[key];
|
|
322
|
+
if (entry === undefined || entry === null)
|
|
323
|
+
return null;
|
|
324
|
+
if (field)
|
|
325
|
+
return get(entry, field) ?? null;
|
|
326
|
+
return entry;
|
|
327
|
+
},
|
|
328
|
+
/**
|
|
329
|
+
* Map an array of keys through a JSON lookup file, collecting a specific
|
|
330
|
+
* field from each matching entry. Non-matching keys are silently skipped.
|
|
331
|
+
* Array-valued fields are flattened into the result.
|
|
332
|
+
*
|
|
333
|
+
* @param filePath - Path to a JSON file (resolved relative to configDir).
|
|
334
|
+
* @param keys - Array of top-level keys to look up.
|
|
335
|
+
* @param field - Dot-path into each looked-up entry.
|
|
336
|
+
* @returns Flat array of resolved values.
|
|
337
|
+
*/
|
|
338
|
+
mapLookup: (filePath, keys, field) => {
|
|
339
|
+
if (!Array.isArray(keys))
|
|
340
|
+
return [];
|
|
341
|
+
const data = loadJson(filePath);
|
|
342
|
+
const results = [];
|
|
343
|
+
for (const k of keys) {
|
|
344
|
+
if (typeof k !== 'string')
|
|
345
|
+
continue;
|
|
346
|
+
const entry = data[k];
|
|
347
|
+
if (entry === undefined || entry === null)
|
|
348
|
+
continue;
|
|
349
|
+
const val = get(entry, field);
|
|
350
|
+
if (val === undefined || val === null)
|
|
351
|
+
continue;
|
|
352
|
+
if (Array.isArray(val)) {
|
|
353
|
+
for (const item of val) {
|
|
354
|
+
results.push(item);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
results.push(val);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
return results;
|
|
362
|
+
},
|
|
363
|
+
...customLib,
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Load custom JsonMap lib functions from file paths.
|
|
368
|
+
*
|
|
369
|
+
* Each module should default-export an object of functions,
|
|
370
|
+
* or use named exports. Only function-valued exports are merged.
|
|
371
|
+
*
|
|
372
|
+
* @param paths - File paths to custom lib modules.
|
|
373
|
+
* @param configDir - Directory to resolve relative paths against.
|
|
374
|
+
* @returns The merged custom lib functions.
|
|
375
|
+
*/
|
|
376
|
+
async function loadCustomMapHelpers(paths, configDir) {
|
|
377
|
+
const merged = {};
|
|
378
|
+
for (const p of paths) {
|
|
379
|
+
const resolved = resolve(configDir, p);
|
|
380
|
+
const mod = (await import(pathToFileURL(resolved).href));
|
|
381
|
+
const fns = typeof mod.default === 'object' && mod.default !== null
|
|
382
|
+
? mod.default
|
|
383
|
+
: mod;
|
|
384
|
+
for (const [key, val] of Object.entries(fns)) {
|
|
385
|
+
if (typeof val === 'function') {
|
|
386
|
+
merged[key] = val;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
return merged;
|
|
391
|
+
}
|
|
392
|
+
/**
|
|
393
|
+
* Apply compiled inference rules to file attributes, returning merged metadata and optional rendered content.
|
|
394
|
+
*
|
|
395
|
+
* Rules are evaluated in order; later rules override earlier ones.
|
|
396
|
+
* If a rule has a `map`, the JsonMap transformation is applied after `set` resolution,
|
|
397
|
+
* and map output overrides set output on conflict.
|
|
398
|
+
*
|
|
399
|
+
* @param compiledRules - The compiled rules to evaluate.
|
|
400
|
+
* @param attributes - The file attributes to match against.
|
|
401
|
+
* @param namedMaps - Optional record of named JsonMap definitions.
|
|
402
|
+
* @param logger - Optional logger for warnings (falls back to console.warn).
|
|
403
|
+
* @param templateEngine - Optional template engine for rendering content templates.
|
|
404
|
+
* @param configDir - Optional config directory for resolving .json map file paths.
|
|
405
|
+
* @returns The merged metadata and optional rendered content.
|
|
406
|
+
*/
|
|
407
|
+
async function applyRules(compiledRules, attributes, namedMaps, logger, templateEngine, configDir, customMapLib) {
|
|
408
|
+
// JsonMap's type definitions expect a generic JsonMapLib shape with unary functions.
|
|
409
|
+
// Our helper functions accept multiple args, which JsonMap supports at runtime.
|
|
410
|
+
const lib = createJsonMapLib(configDir, customMapLib);
|
|
411
|
+
let merged = {};
|
|
412
|
+
let renderedContent = null;
|
|
413
|
+
const log = logger ?? console;
|
|
414
|
+
for (const [ruleIndex, { rule, validate }] of compiledRules.entries()) {
|
|
415
|
+
if (validate(attributes)) {
|
|
416
|
+
// Apply set resolution
|
|
417
|
+
const setOutput = resolveSet(rule.set, attributes);
|
|
418
|
+
merged = { ...merged, ...setOutput };
|
|
419
|
+
// Apply map transformation if present
|
|
420
|
+
if (rule.map) {
|
|
421
|
+
let mapDef;
|
|
422
|
+
// Resolve map reference
|
|
423
|
+
if (typeof rule.map === 'string') {
|
|
424
|
+
if (rule.map.endsWith('.json') && configDir) {
|
|
425
|
+
// File path: load from .json file
|
|
426
|
+
try {
|
|
427
|
+
const mapPath = resolve(configDir, rule.map);
|
|
428
|
+
const raw = readFileSync(mapPath, 'utf-8');
|
|
429
|
+
mapDef = JSON.parse(raw);
|
|
430
|
+
}
|
|
431
|
+
catch (error) {
|
|
432
|
+
log.warn(`Failed to load map file "${rule.map}": ${error instanceof Error ? error.message : String(error)}`);
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
else {
|
|
437
|
+
mapDef = namedMaps?.[rule.map];
|
|
438
|
+
if (!mapDef) {
|
|
439
|
+
log.warn(`Map reference "${rule.map}" not found in named maps. Skipping map transformation.`);
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
else {
|
|
445
|
+
mapDef = rule.map;
|
|
446
|
+
}
|
|
447
|
+
// Execute JsonMap transformation
|
|
448
|
+
try {
|
|
449
|
+
const jsonMap = new JsonMap(mapDef, lib);
|
|
450
|
+
const mapOutput = await jsonMap.transform(attributes);
|
|
451
|
+
if (mapOutput &&
|
|
452
|
+
typeof mapOutput === 'object' &&
|
|
453
|
+
!Array.isArray(mapOutput)) {
|
|
454
|
+
merged = { ...merged, ...mapOutput };
|
|
455
|
+
}
|
|
456
|
+
else {
|
|
457
|
+
log.warn(`JsonMap transformation did not return an object; skipping merge.`);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
catch (error) {
|
|
461
|
+
log.warn(`JsonMap transformation failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
// Render template if present
|
|
465
|
+
if (rule.template && templateEngine) {
|
|
466
|
+
const templateKey = `rule-${String(ruleIndex)}`;
|
|
467
|
+
// Build template context: attributes (with json spread at top) + map output
|
|
468
|
+
const context = {
|
|
469
|
+
...(attributes.json ?? {}),
|
|
470
|
+
...attributes,
|
|
471
|
+
...merged,
|
|
472
|
+
};
|
|
473
|
+
try {
|
|
474
|
+
const result = templateEngine.render(templateKey, context);
|
|
475
|
+
if (result && result.trim()) {
|
|
476
|
+
renderedContent = result;
|
|
477
|
+
}
|
|
478
|
+
else {
|
|
479
|
+
log.warn(`Template for rule ${String(ruleIndex)} rendered empty output. Falling back to raw content.`);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
catch (error) {
|
|
483
|
+
log.warn(`Template render failed for rule ${String(ruleIndex)}: ${error instanceof Error ? error.message : String(error)}. Falling back to raw content.`);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
return { metadata: merged, renderedContent };
|
|
489
|
+
}
|
|
490
|
+
|
|
245
491
|
/**
|
|
246
492
|
* @module templates/helpers
|
|
247
493
|
* Registers built-in Handlebars helpers for content templates.
|
|
@@ -379,7 +625,7 @@ function createHandlebarsInstance() {
|
|
|
379
625
|
async function loadCustomHelpers(hbs, paths, configDir) {
|
|
380
626
|
for (const p of paths) {
|
|
381
627
|
const resolved = resolve(configDir, p);
|
|
382
|
-
const mod = (await import(resolved));
|
|
628
|
+
const mod = (await import(pathToFileURL(resolved).href));
|
|
383
629
|
if (typeof mod.default === 'function') {
|
|
384
630
|
mod.default(hbs);
|
|
385
631
|
}
|
|
@@ -1187,6 +1433,17 @@ const jeevesWatcherConfigSchema = z.object({
|
|
|
1187
1433
|
})
|
|
1188
1434
|
.optional()
|
|
1189
1435
|
.describe('Custom Handlebars helper registration.'),
|
|
1436
|
+
/** Custom JsonMap lib function registration. */
|
|
1437
|
+
mapHelpers: z
|
|
1438
|
+
.object({
|
|
1439
|
+
/** File paths to custom lib modules (each exports functions to merge into the JsonMap lib). */
|
|
1440
|
+
paths: z
|
|
1441
|
+
.array(z.string())
|
|
1442
|
+
.optional()
|
|
1443
|
+
.describe('File paths to JS modules exporting functions to merge into the JsonMap lib.'),
|
|
1444
|
+
})
|
|
1445
|
+
.optional()
|
|
1446
|
+
.describe('Custom JsonMap lib function registration.'),
|
|
1190
1447
|
/** Logging configuration. */
|
|
1191
1448
|
logging: loggingConfigSchema.optional().describe('Logging configuration.'),
|
|
1192
1449
|
/** Timeout in milliseconds for graceful shutdown. */
|
|
@@ -1677,160 +1934,6 @@ async function extractText(filePath, extension) {
|
|
|
1677
1934
|
return extractPlaintext(filePath);
|
|
1678
1935
|
}
|
|
1679
1936
|
|
|
1680
|
-
/**
|
|
1681
|
-
* @module rules/templates
|
|
1682
|
-
* Resolves template variables (`${path.to.value}`) in rule `set` objects against file attributes.
|
|
1683
|
-
*/
|
|
1684
|
-
/**
|
|
1685
|
-
* Resolve `${template.vars}` in a value against the given attributes.
|
|
1686
|
-
*
|
|
1687
|
-
* @param value - The value to resolve.
|
|
1688
|
-
* @param attributes - The file attributes for variable lookup.
|
|
1689
|
-
* @returns The resolved value.
|
|
1690
|
-
*/
|
|
1691
|
-
function resolveTemplateVars(value, attributes) {
|
|
1692
|
-
if (typeof value !== 'string')
|
|
1693
|
-
return value;
|
|
1694
|
-
return value.replace(/\$\{([^}]+)\}/g, (_match, varPath) => {
|
|
1695
|
-
const current = get(attributes, varPath);
|
|
1696
|
-
if (current === null || current === undefined)
|
|
1697
|
-
return '';
|
|
1698
|
-
return typeof current === 'string' ? current : JSON.stringify(current);
|
|
1699
|
-
});
|
|
1700
|
-
}
|
|
1701
|
-
/**
|
|
1702
|
-
* Resolve all template variables in a `set` object.
|
|
1703
|
-
*
|
|
1704
|
-
* @param setObj - The key-value pairs to resolve.
|
|
1705
|
-
* @param attributes - The file attributes for variable lookup.
|
|
1706
|
-
* @returns The resolved key-value pairs.
|
|
1707
|
-
*/
|
|
1708
|
-
function resolveSet(setObj, attributes) {
|
|
1709
|
-
const result = {};
|
|
1710
|
-
for (const [key, value] of Object.entries(setObj)) {
|
|
1711
|
-
result[key] = resolveTemplateVars(value, attributes);
|
|
1712
|
-
}
|
|
1713
|
-
return result;
|
|
1714
|
-
}
|
|
1715
|
-
|
|
1716
|
-
/**
|
|
1717
|
-
* @module rules/apply
|
|
1718
|
-
* Applies compiled inference rules to file attributes, producing merged metadata via template resolution and JsonMap transforms.
|
|
1719
|
-
*/
|
|
1720
|
-
/**
|
|
1721
|
-
* Create the lib object for JsonMap transformations.
|
|
1722
|
-
*
|
|
1723
|
-
* @returns The lib object.
|
|
1724
|
-
*/
|
|
1725
|
-
function createJsonMapLib() {
|
|
1726
|
-
return {
|
|
1727
|
-
split: (str, separator) => str.split(separator),
|
|
1728
|
-
slice: (arr, start, end) => arr.slice(start, end),
|
|
1729
|
-
join: (arr, separator) => arr.join(separator),
|
|
1730
|
-
toLowerCase: (str) => str.toLowerCase(),
|
|
1731
|
-
replace: (str, search, replacement) => str.replace(search, replacement),
|
|
1732
|
-
get: (obj, path) => get(obj, path),
|
|
1733
|
-
};
|
|
1734
|
-
}
|
|
1735
|
-
/**
|
|
1736
|
-
* Apply compiled inference rules to file attributes, returning merged metadata and optional rendered content.
|
|
1737
|
-
*
|
|
1738
|
-
* Rules are evaluated in order; later rules override earlier ones.
|
|
1739
|
-
* If a rule has a `map`, the JsonMap transformation is applied after `set` resolution,
|
|
1740
|
-
* and map output overrides set output on conflict.
|
|
1741
|
-
*
|
|
1742
|
-
* @param compiledRules - The compiled rules to evaluate.
|
|
1743
|
-
* @param attributes - The file attributes to match against.
|
|
1744
|
-
* @param namedMaps - Optional record of named JsonMap definitions.
|
|
1745
|
-
* @param logger - Optional logger for warnings (falls back to console.warn).
|
|
1746
|
-
* @param templateEngine - Optional template engine for rendering content templates.
|
|
1747
|
-
* @param configDir - Optional config directory for resolving .json map file paths.
|
|
1748
|
-
* @returns The merged metadata and optional rendered content.
|
|
1749
|
-
*/
|
|
1750
|
-
async function applyRules(compiledRules, attributes, namedMaps, logger, templateEngine, configDir) {
|
|
1751
|
-
// JsonMap's type definitions expect a generic JsonMapLib shape with unary functions.
|
|
1752
|
-
// Our helper functions accept multiple args, which JsonMap supports at runtime.
|
|
1753
|
-
const lib = createJsonMapLib();
|
|
1754
|
-
let merged = {};
|
|
1755
|
-
let renderedContent = null;
|
|
1756
|
-
const log = logger ?? console;
|
|
1757
|
-
for (const [ruleIndex, { rule, validate }] of compiledRules.entries()) {
|
|
1758
|
-
if (validate(attributes)) {
|
|
1759
|
-
// Apply set resolution
|
|
1760
|
-
const setOutput = resolveSet(rule.set, attributes);
|
|
1761
|
-
merged = { ...merged, ...setOutput };
|
|
1762
|
-
// Apply map transformation if present
|
|
1763
|
-
if (rule.map) {
|
|
1764
|
-
let mapDef;
|
|
1765
|
-
// Resolve map reference
|
|
1766
|
-
if (typeof rule.map === 'string') {
|
|
1767
|
-
if (rule.map.endsWith('.json') && configDir) {
|
|
1768
|
-
// File path: load from .json file
|
|
1769
|
-
try {
|
|
1770
|
-
const mapPath = resolve(configDir, rule.map);
|
|
1771
|
-
const raw = readFileSync(mapPath, 'utf-8');
|
|
1772
|
-
mapDef = JSON.parse(raw);
|
|
1773
|
-
}
|
|
1774
|
-
catch (error) {
|
|
1775
|
-
log.warn(`Failed to load map file "${rule.map}": ${error instanceof Error ? error.message : String(error)}`);
|
|
1776
|
-
continue;
|
|
1777
|
-
}
|
|
1778
|
-
}
|
|
1779
|
-
else {
|
|
1780
|
-
mapDef = namedMaps?.[rule.map];
|
|
1781
|
-
if (!mapDef) {
|
|
1782
|
-
log.warn(`Map reference "${rule.map}" not found in named maps. Skipping map transformation.`);
|
|
1783
|
-
continue;
|
|
1784
|
-
}
|
|
1785
|
-
}
|
|
1786
|
-
}
|
|
1787
|
-
else {
|
|
1788
|
-
mapDef = rule.map;
|
|
1789
|
-
}
|
|
1790
|
-
// Execute JsonMap transformation
|
|
1791
|
-
try {
|
|
1792
|
-
const jsonMap = new JsonMap(mapDef, lib);
|
|
1793
|
-
const mapOutput = await jsonMap.transform(attributes);
|
|
1794
|
-
if (mapOutput &&
|
|
1795
|
-
typeof mapOutput === 'object' &&
|
|
1796
|
-
!Array.isArray(mapOutput)) {
|
|
1797
|
-
merged = { ...merged, ...mapOutput };
|
|
1798
|
-
}
|
|
1799
|
-
else {
|
|
1800
|
-
log.warn(`JsonMap transformation did not return an object; skipping merge.`);
|
|
1801
|
-
}
|
|
1802
|
-
}
|
|
1803
|
-
catch (error) {
|
|
1804
|
-
log.warn(`JsonMap transformation failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1805
|
-
}
|
|
1806
|
-
}
|
|
1807
|
-
// Render template if present
|
|
1808
|
-
if (rule.template && templateEngine) {
|
|
1809
|
-
const templateKey = `rule-${String(ruleIndex)}`;
|
|
1810
|
-
// Build template context: attributes (with json spread at top) + map output
|
|
1811
|
-
const context = {
|
|
1812
|
-
...(attributes.json ?? {}),
|
|
1813
|
-
...attributes,
|
|
1814
|
-
...merged,
|
|
1815
|
-
};
|
|
1816
|
-
try {
|
|
1817
|
-
const result = templateEngine.render(templateKey, context);
|
|
1818
|
-
if (result && result.trim()) {
|
|
1819
|
-
renderedContent = result;
|
|
1820
|
-
}
|
|
1821
|
-
else {
|
|
1822
|
-
log.warn(`Template for rule ${String(ruleIndex)} rendered empty output. Falling back to raw content.`);
|
|
1823
|
-
}
|
|
1824
|
-
}
|
|
1825
|
-
catch (error) {
|
|
1826
|
-
log.warn(`Template render failed for rule ${String(ruleIndex)}: ${error instanceof Error ? error.message : String(error)}. Falling back to raw content.`);
|
|
1827
|
-
}
|
|
1828
|
-
}
|
|
1829
|
-
}
|
|
1830
|
-
}
|
|
1831
|
-
return { metadata: merged, renderedContent };
|
|
1832
|
-
}
|
|
1833
|
-
|
|
1834
1937
|
/**
|
|
1835
1938
|
* @module rules/attributes
|
|
1836
1939
|
* Builds file attribute objects for rule matching. Pure function: derives attributes from path, stats, and extracted data.
|
|
@@ -1921,14 +2024,14 @@ function compileRules(rules) {
|
|
|
1921
2024
|
* @param configDir - Optional config directory for resolving file paths.
|
|
1922
2025
|
* @returns The merged metadata and intermediate data.
|
|
1923
2026
|
*/
|
|
1924
|
-
async function buildMergedMetadata(filePath, compiledRules, metadataDir, maps, logger, templateEngine, configDir) {
|
|
2027
|
+
async function buildMergedMetadata(filePath, compiledRules, metadataDir, maps, logger, templateEngine, configDir, customMapLib) {
|
|
1925
2028
|
const ext = extname(filePath);
|
|
1926
2029
|
const stats = await stat(filePath);
|
|
1927
2030
|
// 1. Extract text and structured data
|
|
1928
2031
|
const extracted = await extractText(filePath, ext);
|
|
1929
2032
|
// 2. Build attributes + apply rules
|
|
1930
2033
|
const attributes = buildAttributes(filePath, stats, extracted.frontmatter, extracted.json);
|
|
1931
|
-
const { metadata: inferred, renderedContent } = await applyRules(compiledRules, attributes, maps, logger, templateEngine, configDir);
|
|
2034
|
+
const { metadata: inferred, renderedContent } = await applyRules(compiledRules, attributes, maps, logger, templateEngine, configDir, customMapLib);
|
|
1932
2035
|
// 3. Read enrichment metadata (merge, enrichment wins)
|
|
1933
2036
|
const enrichment = await readMetadata(filePath, metadataDir);
|
|
1934
2037
|
const metadata = {
|
|
@@ -2041,7 +2144,7 @@ class DocumentProcessor {
|
|
|
2041
2144
|
try {
|
|
2042
2145
|
const ext = extname(filePath);
|
|
2043
2146
|
// 1. Build merged metadata + extract text
|
|
2044
|
-
const { metadata, extracted, renderedContent } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir);
|
|
2147
|
+
const { metadata, extracted, renderedContent } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir, this.config.customMapLib);
|
|
2045
2148
|
// Use rendered template content if available, otherwise raw extracted text
|
|
2046
2149
|
const textToEmbed = renderedContent ?? extracted.text;
|
|
2047
2150
|
if (!textToEmbed.trim()) {
|
|
@@ -2155,7 +2258,7 @@ class DocumentProcessor {
|
|
|
2155
2258
|
return null;
|
|
2156
2259
|
}
|
|
2157
2260
|
// Build merged metadata (lightweight — no embedding)
|
|
2158
|
-
const { metadata } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir);
|
|
2261
|
+
const { metadata } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir, this.config.customMapLib);
|
|
2159
2262
|
// Update all chunk payloads
|
|
2160
2263
|
const totalChunks = getChunkCount(existingPayload);
|
|
2161
2264
|
const ids = chunkIds(filePath, totalChunks);
|
|
@@ -2169,21 +2272,20 @@ class DocumentProcessor {
|
|
|
2169
2272
|
}
|
|
2170
2273
|
}
|
|
2171
2274
|
/**
|
|
2172
|
-
* Update compiled inference rules
|
|
2173
|
-
*
|
|
2174
|
-
* @param compiledRules - The newly compiled rules.
|
|
2175
|
-
*/
|
|
2176
|
-
/**
|
|
2177
|
-
* Update compiled inference rules and optionally the template engine.
|
|
2275
|
+
* Update compiled inference rules, template engine, and custom map lib.
|
|
2178
2276
|
*
|
|
2179
2277
|
* @param compiledRules - The newly compiled rules.
|
|
2180
2278
|
* @param templateEngine - Optional updated template engine.
|
|
2279
|
+
* @param customMapLib - Optional updated custom JsonMap lib functions.
|
|
2181
2280
|
*/
|
|
2182
|
-
updateRules(compiledRules, templateEngine) {
|
|
2281
|
+
updateRules(compiledRules, templateEngine, customMapLib) {
|
|
2183
2282
|
this.compiledRules = compiledRules;
|
|
2184
2283
|
if (templateEngine) {
|
|
2185
2284
|
this.templateEngine = templateEngine;
|
|
2186
2285
|
}
|
|
2286
|
+
if (customMapLib !== undefined) {
|
|
2287
|
+
this.config = { ...this.config, customMapLib };
|
|
2288
|
+
}
|
|
2187
2289
|
this.logger.info({ rules: compiledRules.length }, 'Inference rules updated');
|
|
2188
2290
|
}
|
|
2189
2291
|
}
|
|
@@ -3063,12 +3165,17 @@ class JeevesWatcher {
|
|
|
3063
3165
|
const compiledRules = this.factories.compileRules(this.config.inferenceRules ?? []);
|
|
3064
3166
|
const configDir = this.configPath ? dirname(this.configPath) : '.';
|
|
3065
3167
|
const templateEngine = await buildTemplateEngine(this.config.inferenceRules ?? [], this.config.templates, this.config.templateHelpers?.paths, configDir);
|
|
3168
|
+
// Load custom JsonMap lib functions
|
|
3169
|
+
const customMapLib = this.config.mapHelpers?.paths?.length && configDir
|
|
3170
|
+
? await loadCustomMapHelpers(this.config.mapHelpers.paths, configDir)
|
|
3171
|
+
: undefined;
|
|
3066
3172
|
const processor = this.factories.createDocumentProcessor({
|
|
3067
3173
|
metadataDir: this.config.metadataDir ?? '.jeeves-metadata',
|
|
3068
3174
|
chunkSize: this.config.embedding.chunkSize,
|
|
3069
3175
|
chunkOverlap: this.config.embedding.chunkOverlap,
|
|
3070
3176
|
maps: this.config.maps,
|
|
3071
3177
|
configDir,
|
|
3178
|
+
customMapLib,
|
|
3072
3179
|
}, embeddingProvider, vectorStore, compiledRules, logger, templateEngine);
|
|
3073
3180
|
this.processor = processor;
|
|
3074
3181
|
this.queue = this.factories.createEventQueue({
|
|
@@ -3187,7 +3294,10 @@ class JeevesWatcher {
|
|
|
3187
3294
|
const compiledRules = this.factories.compileRules(newConfig.inferenceRules ?? []);
|
|
3188
3295
|
const reloadConfigDir = dirname(this.configPath);
|
|
3189
3296
|
const newTemplateEngine = await buildTemplateEngine(newConfig.inferenceRules ?? [], newConfig.templates, newConfig.templateHelpers?.paths, reloadConfigDir);
|
|
3190
|
-
|
|
3297
|
+
const newCustomMapLib = newConfig.mapHelpers?.paths?.length && reloadConfigDir
|
|
3298
|
+
? await loadCustomMapHelpers(newConfig.mapHelpers.paths, reloadConfigDir)
|
|
3299
|
+
: undefined;
|
|
3300
|
+
processor.updateRules(compiledRules, newTemplateEngine, newCustomMapLib);
|
|
3191
3301
|
logger.info({ configPath: this.configPath, rules: compiledRules.length }, 'Config reloaded');
|
|
3192
3302
|
}
|
|
3193
3303
|
catch (error) {
|
package/dist/index.d.ts
CHANGED
|
@@ -132,6 +132,9 @@ declare const jeevesWatcherConfigSchema: z.ZodObject<{
|
|
|
132
132
|
templateHelpers: z.ZodOptional<z.ZodObject<{
|
|
133
133
|
paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
134
134
|
}, z.core.$strip>>;
|
|
135
|
+
mapHelpers: z.ZodOptional<z.ZodObject<{
|
|
136
|
+
paths: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
137
|
+
}, z.core.$strip>>;
|
|
135
138
|
logging: z.ZodOptional<z.ZodObject<{
|
|
136
139
|
level: z.ZodOptional<z.ZodString>;
|
|
137
140
|
file: z.ZodOptional<z.ZodString>;
|
|
@@ -366,7 +369,7 @@ interface ApplyRulesResult {
|
|
|
366
369
|
* @param configDir - Optional config directory for resolving .json map file paths.
|
|
367
370
|
* @returns The merged metadata and optional rendered content.
|
|
368
371
|
*/
|
|
369
|
-
declare function applyRules(compiledRules: CompiledRule[], attributes: FileAttributes, namedMaps?: Record<string, JsonMapMap>, logger?: RuleLogger, templateEngine?: TemplateEngine, configDir?: string): Promise<ApplyRulesResult>;
|
|
372
|
+
declare function applyRules(compiledRules: CompiledRule[], attributes: FileAttributes, namedMaps?: Record<string, JsonMapMap>, logger?: RuleLogger, templateEngine?: TemplateEngine, configDir?: string, customMapLib?: Record<string, (...args: unknown[]) => unknown>): Promise<ApplyRulesResult>;
|
|
370
373
|
|
|
371
374
|
/**
|
|
372
375
|
* A point to upsert into the vector store.
|
|
@@ -516,6 +519,8 @@ interface ProcessorConfig {
|
|
|
516
519
|
maps?: Record<string, JsonMapMap>;
|
|
517
520
|
/** Config directory for resolving relative file paths. */
|
|
518
521
|
configDir?: string;
|
|
522
|
+
/** Custom JsonMap lib functions loaded from mapHelpers config. */
|
|
523
|
+
customMapLib?: Record<string, (...args: unknown[]) => unknown>;
|
|
519
524
|
}
|
|
520
525
|
/**
|
|
521
526
|
* Core document processing pipeline.
|
|
@@ -523,7 +528,7 @@ interface ProcessorConfig {
|
|
|
523
528
|
* Handles extracting text, computing embeddings, and syncing with the vector store.
|
|
524
529
|
*/
|
|
525
530
|
declare class DocumentProcessor {
|
|
526
|
-
private
|
|
531
|
+
private config;
|
|
527
532
|
private readonly embeddingProvider;
|
|
528
533
|
private readonly vectorStore;
|
|
529
534
|
private compiledRules;
|
|
@@ -570,17 +575,13 @@ declare class DocumentProcessor {
|
|
|
570
575
|
*/
|
|
571
576
|
processRulesUpdate(filePath: string): Promise<Record<string, unknown> | null>;
|
|
572
577
|
/**
|
|
573
|
-
* Update compiled inference rules
|
|
574
|
-
*
|
|
575
|
-
* @param compiledRules - The newly compiled rules.
|
|
576
|
-
*/
|
|
577
|
-
/**
|
|
578
|
-
* Update compiled inference rules and optionally the template engine.
|
|
578
|
+
* Update compiled inference rules, template engine, and custom map lib.
|
|
579
579
|
*
|
|
580
580
|
* @param compiledRules - The newly compiled rules.
|
|
581
581
|
* @param templateEngine - Optional updated template engine.
|
|
582
|
+
* @param customMapLib - Optional updated custom JsonMap lib functions.
|
|
582
583
|
*/
|
|
583
|
-
updateRules(compiledRules: CompiledRule[], templateEngine?: TemplateEngine): void;
|
|
584
|
+
updateRules(compiledRules: CompiledRule[], templateEngine?: TemplateEngine, customMapLib?: Record<string, (...args: unknown[]) => unknown>): void;
|
|
584
585
|
}
|
|
585
586
|
|
|
586
587
|
/**
|