@karmaniverous/jeeves-watcher 0.4.1 → 0.4.3

package/dist/mjs/index.js

@@ -2,10 +2,11 @@ import Fastify from 'fastify';
 import { readdir, stat, rm, readFile, mkdir, writeFile } from 'node:fs/promises';
 import { resolve, dirname, join, relative, extname, basename } from 'node:path';
 import picomatch from 'picomatch';
-import { omit, capitalize, title, camel, snake, dash, isEqual, get } from 'radash';
+import { omit, get, capitalize, title, camel, snake, dash, isEqual } from 'radash';
 import { createHash } from 'node:crypto';
 import { existsSync, statSync, readdirSync, readFileSync } from 'node:fs';
 import ignore from 'ignore';
+import { JsonMap, jsonMapMapSchema } from '@karmaniverous/jsonmap';
 import Handlebars from 'handlebars';
 import dayjs from 'dayjs';
 import { toMdast } from 'hast-util-to-mdast';
@@ -16,7 +17,6 @@ import { unified } from 'unified';
 import chokidar from 'chokidar';
 import { cosmiconfig } from 'cosmiconfig';
 import { z, ZodError } from 'zod';
-import { jsonMapMapSchema, JsonMap } from '@karmaniverous/jsonmap';
 import { GoogleGenerativeAIEmbeddings } from '@langchain/google-genai';
 import pino from 'pino';
 import { v5 } from 'uuid';
@@ -636,6 +636,251 @@ class GitignoreFilter {
     }
 }
 
+/**
+ * @module rules/templates
+ * Resolves template variables (`${path.to.value}`) in rule `set` objects against file attributes.
+ */
+/**
+ * Resolve `${template.vars}` in a value against the given attributes.
+ *
+ * @param value - The value to resolve.
+ * @param attributes - The file attributes for variable lookup.
+ * @returns The resolved value.
+ */
+function resolveTemplateVars(value, attributes) {
+    if (typeof value !== 'string')
+        return value;
+    return value.replace(/\$\{([^}]+)\}/g, (_match, varPath) => {
+        const current = get(attributes, varPath);
+        if (current === null || current === undefined)
+            return '';
+        return typeof current === 'string' ? current : JSON.stringify(current);
+    });
+}
+/**
+ * Resolve all template variables in a `set` object.
+ *
+ * @param setObj - The key-value pairs to resolve.
+ * @param attributes - The file attributes for variable lookup.
+ * @returns The resolved key-value pairs.
+ */
+function resolveSet(setObj, attributes) {
+    const result = {};
+    for (const [key, value] of Object.entries(setObj)) {
+        result[key] = resolveTemplateVars(value, attributes);
+    }
+    return result;
+}
+
+/**
+ * @module rules/apply
+ * Applies compiled inference rules to file attributes, producing merged metadata via template resolution and JsonMap transforms.
+ */
+/**
+ * Create the lib object for JsonMap transformations.
+ *
+ * @param configDir - Optional config directory for resolving relative file paths in lookups.
+ * @returns The lib object.
+ */
+function createJsonMapLib(configDir, customLib) {
+    // Cache loaded JSON files within a single applyRules invocation.
+    const jsonCache = new Map();
+    const loadJson = (filePath) => {
+        const resolvedPath = configDir ? resolve(configDir, filePath) : filePath;
+        if (!jsonCache.has(resolvedPath)) {
+            const raw = readFileSync(resolvedPath, 'utf-8');
+            jsonCache.set(resolvedPath, JSON.parse(raw));
+        }
+        return jsonCache.get(resolvedPath);
+    };
+    return {
+        split: (str, separator) => str.split(separator),
+        slice: (arr, start, end) => arr.slice(start, end),
+        join: (arr, separator) => arr.join(separator),
+        toLowerCase: (str) => str.toLowerCase(),
+        replace: (str, search, replacement) => str.replace(search, replacement),
+        get: (obj, path) => get(obj, path),
+        /**
+         * Load a JSON file (relative to configDir) and look up a value by key,
+         * optionally drilling into a sub-path.
+         *
+         * @param filePath - Path to a JSON file (resolved relative to configDir).
+         * @param key - Top-level key to look up.
+         * @param field - Optional dot-path into the looked-up entry.
+         * @returns The resolved value, or null if not found.
+         */
+        lookupJson: (filePath, key, field) => {
+            const data = loadJson(filePath);
+            const entry = data[key];
+            if (entry === undefined || entry === null)
+                return null;
+            if (field)
+                return get(entry, field) ?? null;
+            return entry;
+        },
+        /**
+         * Map an array of keys through a JSON lookup file, collecting a specific
+         * field from each matching entry. Non-matching keys are silently skipped.
+         * Array-valued fields are flattened into the result.
+         *
+         * @param filePath - Path to a JSON file (resolved relative to configDir).
+         * @param keys - Array of top-level keys to look up.
+         * @param field - Dot-path into each looked-up entry.
+         * @returns Flat array of resolved values.
+         */
+        mapLookup: (filePath, keys, field) => {
+            if (!Array.isArray(keys))
+                return [];
+            const data = loadJson(filePath);
+            const results = [];
+            for (const k of keys) {
+                if (typeof k !== 'string')
+                    continue;
+                const entry = data[k];
+                if (entry === undefined || entry === null)
+                    continue;
+                const val = get(entry, field);
+                if (val === undefined || val === null)
+                    continue;
+                if (Array.isArray(val)) {
+                    for (const item of val) {
+                        results.push(item);
+                    }
+                }
+                else {
+                    results.push(val);
+                }
+            }
+            return results;
+        },
+        ...customLib,
+    };
+}
+/**
+ * Load custom JsonMap lib functions from file paths.
+ *
+ * Each module should default-export an object of functions,
+ * or use named exports. Only function-valued exports are merged.
+ *
+ * @param paths - File paths to custom lib modules.
+ * @param configDir - Directory to resolve relative paths against.
+ * @returns The merged custom lib functions.
+ */
+async function loadCustomMapHelpers(paths, configDir) {
+    const merged = {};
+    for (const p of paths) {
+        const resolved = resolve(configDir, p);
+        const mod = (await import(resolved));
+        const fns = typeof mod.default === 'object' && mod.default !== null
+            ? mod.default
+            : mod;
+        for (const [key, val] of Object.entries(fns)) {
+            if (typeof val === 'function') {
+                merged[key] = val;
+            }
+        }
+    }
+    return merged;
+}
+/**
+ * Apply compiled inference rules to file attributes, returning merged metadata and optional rendered content.
+ *
+ * Rules are evaluated in order; later rules override earlier ones.
+ * If a rule has a `map`, the JsonMap transformation is applied after `set` resolution,
+ * and map output overrides set output on conflict.
+ *
+ * @param compiledRules - The compiled rules to evaluate.
+ * @param attributes - The file attributes to match against.
+ * @param namedMaps - Optional record of named JsonMap definitions.
+ * @param logger - Optional logger for warnings (falls back to console.warn).
+ * @param templateEngine - Optional template engine for rendering content templates.
+ * @param configDir - Optional config directory for resolving .json map file paths.
+ * @returns The merged metadata and optional rendered content.
+ */
+async function applyRules(compiledRules, attributes, namedMaps, logger, templateEngine, configDir, customMapLib) {
+    // JsonMap's type definitions expect a generic JsonMapLib shape with unary functions.
+    // Our helper functions accept multiple args, which JsonMap supports at runtime.
+    const lib = createJsonMapLib(configDir, customMapLib);
+    let merged = {};
+    let renderedContent = null;
+    const log = logger ?? console;
+    for (const [ruleIndex, { rule, validate }] of compiledRules.entries()) {
+        if (validate(attributes)) {
+            // Apply set resolution
+            const setOutput = resolveSet(rule.set, attributes);
+            merged = { ...merged, ...setOutput };
+            // Apply map transformation if present
+            if (rule.map) {
+                let mapDef;
+                // Resolve map reference
+                if (typeof rule.map === 'string') {
+                    if (rule.map.endsWith('.json') && configDir) {
+                        // File path: load from .json file
+                        try {
+                            const mapPath = resolve(configDir, rule.map);
+                            const raw = readFileSync(mapPath, 'utf-8');
+                            mapDef = JSON.parse(raw);
+                        }
+                        catch (error) {
+                            log.warn(`Failed to load map file "${rule.map}": ${error instanceof Error ? error.message : String(error)}`);
+                            continue;
+                        }
+                    }
+                    else {
+                        mapDef = namedMaps?.[rule.map];
+                        if (!mapDef) {
+                            log.warn(`Map reference "${rule.map}" not found in named maps. Skipping map transformation.`);
+                            continue;
+                        }
+                    }
+                }
+                else {
+                    mapDef = rule.map;
+                }
+                // Execute JsonMap transformation
+                try {
+                    const jsonMap = new JsonMap(mapDef, lib);
+                    const mapOutput = await jsonMap.transform(attributes);
+                    if (mapOutput &&
+                        typeof mapOutput === 'object' &&
+                        !Array.isArray(mapOutput)) {
+                        merged = { ...merged, ...mapOutput };
+                    }
+                    else {
+                        log.warn(`JsonMap transformation did not return an object; skipping merge.`);
+                    }
+                }
+                catch (error) {
+                    log.warn(`JsonMap transformation failed: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+            // Render template if present
+            if (rule.template && templateEngine) {
+                const templateKey = `rule-${String(ruleIndex)}`;
+                // Build template context: attributes (with json spread at top) + map output
+                const context = {
+                    ...(attributes.json ?? {}),
+                    ...attributes,
+                    ...merged,
+                };
+                try {
+                    const result = templateEngine.render(templateKey, context);
+                    if (result && result.trim()) {
+                        renderedContent = result;
+                    }
+                    else {
+                        log.warn(`Template for rule ${String(ruleIndex)} rendered empty output. Falling back to raw content.`);
+                    }
+                }
+                catch (error) {
+                    log.warn(`Template render failed for rule ${String(ruleIndex)}: ${error instanceof Error ? error.message : String(error)}. Falling back to raw content.`);
+                }
+            }
+        }
+    }
+    return { metadata: merged, renderedContent };
+}
+
 /**
  * @module templates/helpers
  * Registers built-in Handlebars helpers for content templates.
@@ -1143,11 +1388,11 @@ const jeevesWatcherConfigSchema = z.object({
         .array(inferenceRuleSchema)
         .optional()
         .describe('Rules for inferring metadata from file attributes.'),
-    /** Reusable named JsonMap transformations. */
+    /** Reusable named JsonMap transformations (inline objects or .json file paths). */
     maps: z
-        .record(z.string(), jsonMapMapSchema)
+        .record(z.string(), jsonMapMapSchema.or(z.string()))
         .optional()
-        .describe('Reusable named JsonMap transformations.'),
+        .describe('Reusable named JsonMap transformations (inline definition or .json file path resolved relative to config directory).'),
     /** Reusable named Handlebars templates (inline strings or .hbs/.handlebars file paths). */
     templates: z
         .record(z.string(), z.string())
@@ -1164,6 +1409,17 @@ const jeevesWatcherConfigSchema = z.object({
     })
         .optional()
         .describe('Custom Handlebars helper registration.'),
+    /** Custom JsonMap lib function registration. */
+    mapHelpers: z
+        .object({
+        /** File paths to custom lib modules (each exports functions to merge into the JsonMap lib). */
+        paths: z
+            .array(z.string())
+            .optional()
+            .describe('File paths to JS modules exporting functions to merge into the JsonMap lib.'),
+    })
+        .optional()
+        .describe('Custom JsonMap lib function registration.'),
     /** Logging configuration. */
     logging: loggingConfigSchema.optional().describe('Logging configuration.'),
     /** Timeout in milliseconds for graceful shutdown. */
@@ -1261,6 +1517,18 @@ async function loadConfig(configPath) {
     }
     try {
         const validated = jeevesWatcherConfigSchema.parse(result.config);
+        // Resolve file-path map references relative to config directory.
+        // After this block, all map values are inline JsonMapMap objects.
+        if (validated.maps) {
+            const configDir = dirname(result.filepath);
+            for (const [name, value] of Object.entries(validated.maps)) {
+                if (typeof value === 'string') {
+                    const mapPath = resolve(configDir, value);
+                    const raw = readFileSync(mapPath, 'utf-8');
+                    validated.maps[name] = JSON.parse(raw);
+                }
+            }
+        }
         const withDefaults = applyDefaults(validated);
         return substituteEnvVars(withDefaults);
     }
@@ -1642,160 +1910,6 @@ async function extractText(filePath, extension) {
     return extractPlaintext(filePath);
 }
 
-/**
- * @module rules/templates
- * Resolves template variables (`${path.to.value}`) in rule `set` objects against file attributes.
- */
-/**
- * Resolve `${template.vars}` in a value against the given attributes.
- *
- * @param value - The value to resolve.
- * @param attributes - The file attributes for variable lookup.
- * @returns The resolved value.
- */
-function resolveTemplateVars(value, attributes) {
-    if (typeof value !== 'string')
-        return value;
-    return value.replace(/\$\{([^}]+)\}/g, (_match, varPath) => {
-        const current = get(attributes, varPath);
-        if (current === null || current === undefined)
-            return '';
-        return typeof current === 'string' ? current : JSON.stringify(current);
-    });
-}
-/**
- * Resolve all template variables in a `set` object.
- *
- * @param setObj - The key-value pairs to resolve.
- * @param attributes - The file attributes for variable lookup.
- * @returns The resolved key-value pairs.
- */
-function resolveSet(setObj, attributes) {
-    const result = {};
-    for (const [key, value] of Object.entries(setObj)) {
-        result[key] = resolveTemplateVars(value, attributes);
-    }
-    return result;
-}
-
-/**
- * @module rules/apply
- * Applies compiled inference rules to file attributes, producing merged metadata via template resolution and JsonMap transforms.
- */
-/**
- * Create the lib object for JsonMap transformations.
- *
- * @returns The lib object.
- */
-function createJsonMapLib() {
-    return {
-        split: (str, separator) => str.split(separator),
-        slice: (arr, start, end) => arr.slice(start, end),
-        join: (arr, separator) => arr.join(separator),
-        toLowerCase: (str) => str.toLowerCase(),
-        replace: (str, search, replacement) => str.replace(search, replacement),
-        get: (obj, path) => get(obj, path),
-    };
-}
-/**
- * Apply compiled inference rules to file attributes, returning merged metadata and optional rendered content.
- *
- * Rules are evaluated in order; later rules override earlier ones.
- * If a rule has a `map`, the JsonMap transformation is applied after `set` resolution,
- * and map output overrides set output on conflict.
- *
- * @param compiledRules - The compiled rules to evaluate.
- * @param attributes - The file attributes to match against.
- * @param namedMaps - Optional record of named JsonMap definitions.
- * @param logger - Optional logger for warnings (falls back to console.warn).
- * @param templateEngine - Optional template engine for rendering content templates.
- * @param configDir - Optional config directory for resolving .json map file paths.
- * @returns The merged metadata and optional rendered content.
- */
-async function applyRules(compiledRules, attributes, namedMaps, logger, templateEngine, configDir) {
-    // JsonMap's type definitions expect a generic JsonMapLib shape with unary functions.
-    // Our helper functions accept multiple args, which JsonMap supports at runtime.
-    const lib = createJsonMapLib();
-    let merged = {};
-    let renderedContent = null;
-    const log = logger ?? console;
-    for (const [ruleIndex, { rule, validate }] of compiledRules.entries()) {
-        if (validate(attributes)) {
-            // Apply set resolution
-            const setOutput = resolveSet(rule.set, attributes);
-            merged = { ...merged, ...setOutput };
-            // Apply map transformation if present
-            if (rule.map) {
-                let mapDef;
-                // Resolve map reference
-                if (typeof rule.map === 'string') {
-                    if (rule.map.endsWith('.json') && configDir) {
-                        // File path: load from .json file
-                        try {
-                            const mapPath = resolve(configDir, rule.map);
-                            const raw = readFileSync(mapPath, 'utf-8');
-                            mapDef = JSON.parse(raw);
-                        }
-                        catch (error) {
-                            log.warn(`Failed to load map file "${rule.map}": ${error instanceof Error ? error.message : String(error)}`);
-                            continue;
-                        }
-                    }
-                    else {
-                        mapDef = namedMaps?.[rule.map];
-                        if (!mapDef) {
-                            log.warn(`Map reference "${rule.map}" not found in named maps. Skipping map transformation.`);
-                            continue;
-                        }
-                    }
-                }
-                else {
-                    mapDef = rule.map;
-                }
-                // Execute JsonMap transformation
-                try {
-                    const jsonMap = new JsonMap(mapDef, lib);
-                    const mapOutput = await jsonMap.transform(attributes);
-                    if (mapOutput &&
-                        typeof mapOutput === 'object' &&
-                        !Array.isArray(mapOutput)) {
-                        merged = { ...merged, ...mapOutput };
-                    }
-                    else {
-                        log.warn(`JsonMap transformation did not return an object; skipping merge.`);
-                    }
-                }
-                catch (error) {
-                    log.warn(`JsonMap transformation failed: ${error instanceof Error ? error.message : String(error)}`);
-                }
-            }
-            // Render template if present
-            if (rule.template && templateEngine) {
-                const templateKey = `rule-${String(ruleIndex)}`;
-                // Build template context: attributes (with json spread at top) + map output
-                const context = {
-                    ...(attributes.json ?? {}),
-                    ...attributes,
-                    ...merged,
-                };
-                try {
-                    const result = templateEngine.render(templateKey, context);
-                    if (result && result.trim()) {
-                        renderedContent = result;
-                    }
-                    else {
-                        log.warn(`Template for rule ${String(ruleIndex)} rendered empty output. Falling back to raw content.`);
-                    }
-                }
-                catch (error) {
-                    log.warn(`Template render failed for rule ${String(ruleIndex)}: ${error instanceof Error ? error.message : String(error)}. Falling back to raw content.`);
-                }
-            }
-        }
-    }
-    return { metadata: merged, renderedContent };
-}
-
 /**
  * @module rules/attributes
  * Builds file attribute objects for rule matching. Pure function: derives attributes from path, stats, and extracted data.
@@ -1886,14 +2000,14 @@ function compileRules(rules) {
  * @param configDir - Optional config directory for resolving file paths.
  * @returns The merged metadata and intermediate data.
  */
-async function buildMergedMetadata(filePath, compiledRules, metadataDir, maps, logger, templateEngine, configDir) {
+async function buildMergedMetadata(filePath, compiledRules, metadataDir, maps, logger, templateEngine, configDir, customMapLib) {
     const ext = extname(filePath);
     const stats = await stat(filePath);
     // 1. Extract text and structured data
     const extracted = await extractText(filePath, ext);
     // 2. Build attributes + apply rules
     const attributes = buildAttributes(filePath, stats, extracted.frontmatter, extracted.json);
-    const { metadata: inferred, renderedContent } = await applyRules(compiledRules, attributes, maps, logger, templateEngine, configDir);
+    const { metadata: inferred, renderedContent } = await applyRules(compiledRules, attributes, maps, logger, templateEngine, configDir, customMapLib);
     // 3. Read enrichment metadata (merge, enrichment wins)
     const enrichment = await readMetadata(filePath, metadataDir);
     const metadata = {
@@ -2006,7 +2120,7 @@ class DocumentProcessor {
         try {
             const ext = extname(filePath);
             // 1. Build merged metadata + extract text
-            const { metadata, extracted, renderedContent } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir);
+            const { metadata, extracted, renderedContent } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir, this.config.customMapLib);
             // Use rendered template content if available, otherwise raw extracted text
             const textToEmbed = renderedContent ?? extracted.text;
             if (!textToEmbed.trim()) {
@@ -2120,7 +2234,7 @@ class DocumentProcessor {
                 return null;
             }
             // Build merged metadata (lightweight — no embedding)
-            const { metadata } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir);
+            const { metadata } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir, this.config.customMapLib);
             // Update all chunk payloads
             const totalChunks = getChunkCount(existingPayload);
             const ids = chunkIds(filePath, totalChunks);
@@ -2134,21 +2248,20 @@ class DocumentProcessor {
         }
     }
     /**
-     * Update compiled inference rules for subsequent file processing.
-     *
-     * @param compiledRules - The newly compiled rules.
-     */
-    /**
-     * Update compiled inference rules and optionally the template engine.
+     * Update compiled inference rules, template engine, and custom map lib.
      *
      * @param compiledRules - The newly compiled rules.
      * @param templateEngine - Optional updated template engine.
+     * @param customMapLib - Optional updated custom JsonMap lib functions.
      */
-    updateRules(compiledRules, templateEngine) {
+    updateRules(compiledRules, templateEngine, customMapLib) {
         this.compiledRules = compiledRules;
         if (templateEngine) {
             this.templateEngine = templateEngine;
         }
+        if (customMapLib !== undefined) {
+            this.config = { ...this.config, customMapLib };
+        }
         this.logger.info({ rules: compiledRules.length }, 'Inference rules updated');
     }
 }
@@ -3028,12 +3141,17 @@ class JeevesWatcher {
         const compiledRules = this.factories.compileRules(this.config.inferenceRules ?? []);
         const configDir = this.configPath ? dirname(this.configPath) : '.';
         const templateEngine = await buildTemplateEngine(this.config.inferenceRules ?? [], this.config.templates, this.config.templateHelpers?.paths, configDir);
+        // Load custom JsonMap lib functions
+        const customMapLib = this.config.mapHelpers?.paths?.length && configDir
+            ? await loadCustomMapHelpers(this.config.mapHelpers.paths, configDir)
+            : undefined;
         const processor = this.factories.createDocumentProcessor({
             metadataDir: this.config.metadataDir ?? '.jeeves-metadata',
             chunkSize: this.config.embedding.chunkSize,
             chunkOverlap: this.config.embedding.chunkOverlap,
             maps: this.config.maps,
             configDir,
+            customMapLib,
         }, embeddingProvider, vectorStore, compiledRules, logger, templateEngine);
         this.processor = processor;
         this.queue = this.factories.createEventQueue({
@@ -3152,7 +3270,10 @@ class JeevesWatcher {
             const compiledRules = this.factories.compileRules(newConfig.inferenceRules ?? []);
             const reloadConfigDir = dirname(this.configPath);
             const newTemplateEngine = await buildTemplateEngine(newConfig.inferenceRules ?? [], newConfig.templates, newConfig.templateHelpers?.paths, reloadConfigDir);
-            processor.updateRules(compiledRules, newTemplateEngine);
+            const newCustomMapLib = newConfig.mapHelpers?.paths?.length && reloadConfigDir
+                ? await loadCustomMapHelpers(newConfig.mapHelpers.paths, reloadConfigDir)
+                : undefined;
+            processor.updateRules(compiledRules, newTemplateEngine, newCustomMapLib);
             logger.info({ configPath: this.configPath, rules: compiledRules.length }, 'Config reloaded');
         }
         catch (error) {

package/package.json

@@ -185,5 +185,5 @@
   },
   "type": "module",
   "types": "dist/index.d.ts",
-  "version": "0.4.1"
+  "version": "0.4.3"
 }