@karmaniverous/jeeves-watcher 0.4.1 → 0.4.3

package/dist/index.iife.js

@@ -1,4 +1,4 @@
-(function (exports, Fastify, promises, node_path, picomatch, radash, node_crypto, node_fs, ignore, Handlebars, dayjs, hastUtilToMdast, mdastUtilFromAdf, mdastUtilToMarkdown, rehypeParse, unified, chokidar, cosmiconfig, zod, jsonmap, googleGenai, pino, uuid, cheerio, yaml, mammoth, Ajv, addFormats, textsplitters, jsClientRest) {
+(function (exports, Fastify, promises, node_path, picomatch, radash, node_crypto, node_fs, ignore, jsonmap, Handlebars, dayjs, hastUtilToMdast, mdastUtilFromAdf, mdastUtilToMarkdown, rehypeParse, unified, chokidar, cosmiconfig, zod, googleGenai, pino, uuid, cheerio, yaml, mammoth, Ajv, addFormats, textsplitters, jsClientRest) {
     'use strict';
 
     function _interopNamespaceDefault(e) {
@@ -628,6 +628,251 @@
         }
     }
 
+    /**
+     * @module rules/templates
+     * Resolves template variables (`${path.to.value}`) in rule `set` objects against file attributes.
+     */
+    /**
+     * Resolve `${template.vars}` in a value against the given attributes.
+     *
+     * @param value - The value to resolve.
+     * @param attributes - The file attributes for variable lookup.
+     * @returns The resolved value.
+     */
+    function resolveTemplateVars(value, attributes) {
+        if (typeof value !== 'string')
+            return value;
+        return value.replace(/\$\{([^}]+)\}/g, (_match, varPath) => {
+            const current = radash.get(attributes, varPath);
+            if (current === null || current === undefined)
+                return '';
+            return typeof current === 'string' ? current : JSON.stringify(current);
+        });
+    }
+    /**
+     * Resolve all template variables in a `set` object.
+     *
+     * @param setObj - The key-value pairs to resolve.
+     * @param attributes - The file attributes for variable lookup.
+     * @returns The resolved key-value pairs.
+     */
+    function resolveSet(setObj, attributes) {
+        const result = {};
+        for (const [key, value] of Object.entries(setObj)) {
+            result[key] = resolveTemplateVars(value, attributes);
+        }
+        return result;
+    }
+
+    /**
+     * @module rules/apply
+     * Applies compiled inference rules to file attributes, producing merged metadata via template resolution and JsonMap transforms.
+     */
+    /**
+     * Create the lib object for JsonMap transformations.
+     *
+     * @param configDir - Optional config directory for resolving relative file paths in lookups.
+     * @returns The lib object.
+     */
+    function createJsonMapLib(configDir, customLib) {
+        // Cache loaded JSON files within a single applyRules invocation.
+        const jsonCache = new Map();
+        const loadJson = (filePath) => {
+            const resolvedPath = configDir ? node_path.resolve(configDir, filePath) : filePath;
+            if (!jsonCache.has(resolvedPath)) {
+                const raw = node_fs.readFileSync(resolvedPath, 'utf-8');
+                jsonCache.set(resolvedPath, JSON.parse(raw));
+            }
+            return jsonCache.get(resolvedPath);
+        };
+        return {
+            split: (str, separator) => str.split(separator),
+            slice: (arr, start, end) => arr.slice(start, end),
+            join: (arr, separator) => arr.join(separator),
+            toLowerCase: (str) => str.toLowerCase(),
+            replace: (str, search, replacement) => str.replace(search, replacement),
+            get: (obj, path) => radash.get(obj, path),
+            /**
+             * Load a JSON file (relative to configDir) and look up a value by key,
+             * optionally drilling into a sub-path.
+             *
+             * @param filePath - Path to a JSON file (resolved relative to configDir).
+             * @param key - Top-level key to look up.
+             * @param field - Optional dot-path into the looked-up entry.
+             * @returns The resolved value, or null if not found.
+             */
+            lookupJson: (filePath, key, field) => {
+                const data = loadJson(filePath);
+                const entry = data[key];
+                if (entry === undefined || entry === null)
+                    return null;
+                if (field)
+                    return radash.get(entry, field) ?? null;
+                return entry;
+            },
+            /**
+             * Map an array of keys through a JSON lookup file, collecting a specific
+             * field from each matching entry. Non-matching keys are silently skipped.
+             * Array-valued fields are flattened into the result.
+             *
+             * @param filePath - Path to a JSON file (resolved relative to configDir).
+             * @param keys - Array of top-level keys to look up.
+             * @param field - Dot-path into each looked-up entry.
+             * @returns Flat array of resolved values.
+             */
+            mapLookup: (filePath, keys, field) => {
+                if (!Array.isArray(keys))
+                    return [];
+                const data = loadJson(filePath);
+                const results = [];
+                for (const k of keys) {
+                    if (typeof k !== 'string')
+                        continue;
+                    const entry = data[k];
+                    if (entry === undefined || entry === null)
+                        continue;
+                    const val = radash.get(entry, field);
+                    if (val === undefined || val === null)
+                        continue;
+                    if (Array.isArray(val)) {
+                        for (const item of val) {
+                            results.push(item);
+                        }
+                    }
+                    else {
+                        results.push(val);
+                    }
+                }
+                return results;
+            },
+            ...customLib,
+        };
+    }
+    /**
+     * Load custom JsonMap lib functions from file paths.
+     *
+     * Each module should default-export an object of functions,
+     * or use named exports. Only function-valued exports are merged.
+     *
+     * @param paths - File paths to custom lib modules.
+     * @param configDir - Directory to resolve relative paths against.
+     * @returns The merged custom lib functions.
+     */
+    async function loadCustomMapHelpers(paths, configDir) {
+        const merged = {};
+        for (const p of paths) {
+            const resolved = node_path.resolve(configDir, p);
+            const mod = (await import(resolved));
+            const fns = typeof mod.default === 'object' && mod.default !== null
+                ? mod.default
+                : mod;
+            for (const [key, val] of Object.entries(fns)) {
+                if (typeof val === 'function') {
+                    merged[key] = val;
+                }
+            }
+        }
+        return merged;
+    }
+    /**
+     * Apply compiled inference rules to file attributes, returning merged metadata and optional rendered content.
+     *
+     * Rules are evaluated in order; later rules override earlier ones.
+     * If a rule has a `map`, the JsonMap transformation is applied after `set` resolution,
+     * and map output overrides set output on conflict.
+     *
+     * @param compiledRules - The compiled rules to evaluate.
+     * @param attributes - The file attributes to match against.
+     * @param namedMaps - Optional record of named JsonMap definitions.
+     * @param logger - Optional logger for warnings (falls back to console.warn).
+     * @param templateEngine - Optional template engine for rendering content templates.
+     * @param configDir - Optional config directory for resolving .json map file paths.
+     * @returns The merged metadata and optional rendered content.
+     */
+    async function applyRules(compiledRules, attributes, namedMaps, logger, templateEngine, configDir, customMapLib) {
+        // JsonMap's type definitions expect a generic JsonMapLib shape with unary functions.
+        // Our helper functions accept multiple args, which JsonMap supports at runtime.
+        const lib = createJsonMapLib(configDir, customMapLib);
+        let merged = {};
+        let renderedContent = null;
+        const log = logger ?? console;
+        for (const [ruleIndex, { rule, validate }] of compiledRules.entries()) {
+            if (validate(attributes)) {
+                // Apply set resolution
+                const setOutput = resolveSet(rule.set, attributes);
+                merged = { ...merged, ...setOutput };
+                // Apply map transformation if present
+                if (rule.map) {
+                    let mapDef;
+                    // Resolve map reference
+                    if (typeof rule.map === 'string') {
+                        if (rule.map.endsWith('.json') && configDir) {
+                            // File path: load from .json file
+                            try {
+                                const mapPath = node_path.resolve(configDir, rule.map);
+                                const raw = node_fs.readFileSync(mapPath, 'utf-8');
+                                mapDef = JSON.parse(raw);
+                            }
+                            catch (error) {
+                                log.warn(`Failed to load map file "${rule.map}": ${error instanceof Error ? error.message : String(error)}`);
+                                continue;
+                            }
+                        }
+                        else {
+                            mapDef = namedMaps?.[rule.map];
+                            if (!mapDef) {
+                                log.warn(`Map reference "${rule.map}" not found in named maps. Skipping map transformation.`);
+                                continue;
+                            }
+                        }
+                    }
+                    else {
+                        mapDef = rule.map;
+                    }
+                    // Execute JsonMap transformation
+                    try {
+                        const jsonMap = new jsonmap.JsonMap(mapDef, lib);
+                        const mapOutput = await jsonMap.transform(attributes);
+                        if (mapOutput &&
+                            typeof mapOutput === 'object' &&
+                            !Array.isArray(mapOutput)) {
+                            merged = { ...merged, ...mapOutput };
+                        }
+                        else {
+                            log.warn(`JsonMap transformation did not return an object; skipping merge.`);
+                        }
+                    }
+                    catch (error) {
+                        log.warn(`JsonMap transformation failed: ${error instanceof Error ? error.message : String(error)}`);
+                    }
+                }
+                // Render template if present
+                if (rule.template && templateEngine) {
+                    const templateKey = `rule-${String(ruleIndex)}`;
+                    // Build template context: attributes (with json spread at top) + map output
+                    const context = {
+                        ...(attributes.json ?? {}),
+                        ...attributes,
+                        ...merged,
+                    };
+                    try {
+                        const result = templateEngine.render(templateKey, context);
+                        if (result && result.trim()) {
+                            renderedContent = result;
+                        }
+                        else {
+                            log.warn(`Template for rule ${String(ruleIndex)} rendered empty output. Falling back to raw content.`);
+                        }
+                    }
+                    catch (error) {
+                        log.warn(`Template render failed for rule ${String(ruleIndex)}: ${error instanceof Error ? error.message : String(error)}. Falling back to raw content.`);
+                    }
+                }
+            }
+        }
+        return { metadata: merged, renderedContent };
+    }
+
     /**
      * @module templates/helpers
      * Registers built-in Handlebars helpers for content templates.
@@ -1135,11 +1380,11 @@
             .array(inferenceRuleSchema)
             .optional()
             .describe('Rules for inferring metadata from file attributes.'),
-        /** Reusable named JsonMap transformations. */
+        /** Reusable named JsonMap transformations (inline objects or .json file paths). */
         maps: zod.z
-            .record(zod.z.string(), jsonmap.jsonMapMapSchema)
+            .record(zod.z.string(), jsonmap.jsonMapMapSchema.or(zod.z.string()))
             .optional()
-            .describe('Reusable named JsonMap transformations.'),
+            .describe('Reusable named JsonMap transformations (inline definition or .json file path resolved relative to config directory).'),
         /** Reusable named Handlebars templates (inline strings or .hbs/.handlebars file paths). */
         templates: zod.z
             .record(zod.z.string(), zod.z.string())
@@ -1156,6 +1401,17 @@
         })
             .optional()
             .describe('Custom Handlebars helper registration.'),
+        /** Custom JsonMap lib function registration. */
+        mapHelpers: zod.z
+            .object({
+            /** File paths to custom lib modules (each exports functions to merge into the JsonMap lib). */
+            paths: zod.z
+                .array(zod.z.string())
+                .optional()
+                .describe('File paths to JS modules exporting functions to merge into the JsonMap lib.'),
+        })
+            .optional()
+            .describe('Custom JsonMap lib function registration.'),
         /** Logging configuration. */
         logging: loggingConfigSchema.optional().describe('Logging configuration.'),
         /** Timeout in milliseconds for graceful shutdown. */
@@ -1253,6 +1509,18 @@
         }
         try {
             const validated = jeevesWatcherConfigSchema.parse(result.config);
+            // Resolve file-path map references relative to config directory.
+            // After this block, all map values are inline JsonMapMap objects.
+            if (validated.maps) {
+                const configDir = node_path.dirname(result.filepath);
+                for (const [name, value] of Object.entries(validated.maps)) {
+                    if (typeof value === 'string') {
+                        const mapPath = node_path.resolve(configDir, value);
+                        const raw = node_fs.readFileSync(mapPath, 'utf-8');
+                        validated.maps[name] = JSON.parse(raw);
+                    }
+                }
+            }
             const withDefaults = applyDefaults(validated);
             return substituteEnvVars(withDefaults);
         }
@@ -1634,160 +1902,6 @@
         return extractPlaintext(filePath);
     }
 
-    /**
-     * @module rules/templates
-     * Resolves template variables (`${path.to.value}`) in rule `set` objects against file attributes.
-     */
-    /**
-     * Resolve `${template.vars}` in a value against the given attributes.
-     *
-     * @param value - The value to resolve.
-     * @param attributes - The file attributes for variable lookup.
-     * @returns The resolved value.
-     */
-    function resolveTemplateVars(value, attributes) {
-        if (typeof value !== 'string')
-            return value;
-        return value.replace(/\$\{([^}]+)\}/g, (_match, varPath) => {
-            const current = radash.get(attributes, varPath);
-            if (current === null || current === undefined)
-                return '';
-            return typeof current === 'string' ? current : JSON.stringify(current);
-        });
-    }
-    /**
-     * Resolve all template variables in a `set` object.
-     *
-     * @param setObj - The key-value pairs to resolve.
-     * @param attributes - The file attributes for variable lookup.
-     * @returns The resolved key-value pairs.
-     */
-    function resolveSet(setObj, attributes) {
-        const result = {};
-        for (const [key, value] of Object.entries(setObj)) {
-            result[key] = resolveTemplateVars(value, attributes);
-        }
-        return result;
-    }
-
-    /**
-     * @module rules/apply
-     * Applies compiled inference rules to file attributes, producing merged metadata via template resolution and JsonMap transforms.
-     */
-    /**
-     * Create the lib object for JsonMap transformations.
-     *
-     * @returns The lib object.
-     */
-    function createJsonMapLib() {
-        return {
-            split: (str, separator) => str.split(separator),
-            slice: (arr, start, end) => arr.slice(start, end),
-            join: (arr, separator) => arr.join(separator),
-            toLowerCase: (str) => str.toLowerCase(),
-            replace: (str, search, replacement) => str.replace(search, replacement),
-            get: (obj, path) => radash.get(obj, path),
-        };
-    }
-    /**
-     * Apply compiled inference rules to file attributes, returning merged metadata and optional rendered content.
-     *
-     * Rules are evaluated in order; later rules override earlier ones.
-     * If a rule has a `map`, the JsonMap transformation is applied after `set` resolution,
-     * and map output overrides set output on conflict.
-     *
-     * @param compiledRules - The compiled rules to evaluate.
-     * @param attributes - The file attributes to match against.
-     * @param namedMaps - Optional record of named JsonMap definitions.
-     * @param logger - Optional logger for warnings (falls back to console.warn).
-     * @param templateEngine - Optional template engine for rendering content templates.
-     * @param configDir - Optional config directory for resolving .json map file paths.
-     * @returns The merged metadata and optional rendered content.
-     */
-    async function applyRules(compiledRules, attributes, namedMaps, logger, templateEngine, configDir) {
-        // JsonMap's type definitions expect a generic JsonMapLib shape with unary functions.
-        // Our helper functions accept multiple args, which JsonMap supports at runtime.
-        const lib = createJsonMapLib();
-        let merged = {};
-        let renderedContent = null;
-        const log = logger ?? console;
-        for (const [ruleIndex, { rule, validate }] of compiledRules.entries()) {
-            if (validate(attributes)) {
-                // Apply set resolution
-                const setOutput = resolveSet(rule.set, attributes);
-                merged = { ...merged, ...setOutput };
-                // Apply map transformation if present
-                if (rule.map) {
-                    let mapDef;
-                    // Resolve map reference
-                    if (typeof rule.map === 'string') {
-                        if (rule.map.endsWith('.json') && configDir) {
-                            // File path: load from .json file
-                            try {
-                                const mapPath = node_path.resolve(configDir, rule.map);
-                                const raw = node_fs.readFileSync(mapPath, 'utf-8');
-                                mapDef = JSON.parse(raw);
-                            }
-                            catch (error) {
-                                log.warn(`Failed to load map file "${rule.map}": ${error instanceof Error ? error.message : String(error)}`);
-                                continue;
-                            }
-                        }
-                        else {
-                            mapDef = namedMaps?.[rule.map];
-                            if (!mapDef) {
-                                log.warn(`Map reference "${rule.map}" not found in named maps. Skipping map transformation.`);
-                                continue;
-                            }
-                        }
-                    }
-                    else {
-                        mapDef = rule.map;
-                    }
-                    // Execute JsonMap transformation
-                    try {
-                        const jsonMap = new jsonmap.JsonMap(mapDef, lib);
-                        const mapOutput = await jsonMap.transform(attributes);
-                        if (mapOutput &&
-                            typeof mapOutput === 'object' &&
-                            !Array.isArray(mapOutput)) {
-                            merged = { ...merged, ...mapOutput };
-                        }
-                        else {
-                            log.warn(`JsonMap transformation did not return an object; skipping merge.`);
-                        }
-                    }
-                    catch (error) {
-                        log.warn(`JsonMap transformation failed: ${error instanceof Error ? error.message : String(error)}`);
-                    }
-                }
-                // Render template if present
-                if (rule.template && templateEngine) {
-                    const templateKey = `rule-${String(ruleIndex)}`;
-                    // Build template context: attributes (with json spread at top) + map output
-                    const context = {
-                        ...(attributes.json ?? {}),
-                        ...attributes,
-                        ...merged,
-                    };
-                    try {
-                        const result = templateEngine.render(templateKey, context);
-                        if (result && result.trim()) {
-                            renderedContent = result;
-                        }
-                        else {
-                            log.warn(`Template for rule ${String(ruleIndex)} rendered empty output. Falling back to raw content.`);
-                        }
-                    }
-                    catch (error) {
-                        log.warn(`Template render failed for rule ${String(ruleIndex)}: ${error instanceof Error ? error.message : String(error)}. Falling back to raw content.`);
-                    }
-                }
-            }
-        }
-        return { metadata: merged, renderedContent };
-    }
-
     /**
      * @module rules/attributes
      * Builds file attribute objects for rule matching. Pure function: derives attributes from path, stats, and extracted data.
@@ -1878,14 +1992,14 @@
      * @param configDir - Optional config directory for resolving file paths.
      * @returns The merged metadata and intermediate data.
      */
-    async function buildMergedMetadata(filePath, compiledRules, metadataDir, maps, logger, templateEngine, configDir) {
+    async function buildMergedMetadata(filePath, compiledRules, metadataDir, maps, logger, templateEngine, configDir, customMapLib) {
         const ext = node_path.extname(filePath);
         const stats = await promises.stat(filePath);
         // 1. Extract text and structured data
         const extracted = await extractText(filePath, ext);
         // 2. Build attributes + apply rules
         const attributes = buildAttributes(filePath, stats, extracted.frontmatter, extracted.json);
-        const { metadata: inferred, renderedContent } = await applyRules(compiledRules, attributes, maps, logger, templateEngine, configDir);
+        const { metadata: inferred, renderedContent } = await applyRules(compiledRules, attributes, maps, logger, templateEngine, configDir, customMapLib);
         // 3. Read enrichment metadata (merge, enrichment wins)
         const enrichment = await readMetadata(filePath, metadataDir);
         const metadata = {
@@ -1998,7 +2112,7 @@
             try {
                 const ext = node_path.extname(filePath);
                 // 1. Build merged metadata + extract text
-                const { metadata, extracted, renderedContent } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir);
+                const { metadata, extracted, renderedContent } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir, this.config.customMapLib);
                 // Use rendered template content if available, otherwise raw extracted text
                 const textToEmbed = renderedContent ?? extracted.text;
                 if (!textToEmbed.trim()) {
@@ -2112,7 +2226,7 @@
                     return null;
                 }
                 // Build merged metadata (lightweight — no embedding)
-                const { metadata } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir);
+                const { metadata } = await buildMergedMetadata(filePath, this.compiledRules, this.config.metadataDir, this.config.maps, this.logger, this.templateEngine, this.config.configDir, this.config.customMapLib);
                 // Update all chunk payloads
                 const totalChunks = getChunkCount(existingPayload);
                 const ids = chunkIds(filePath, totalChunks);
@@ -2126,21 +2240,20 @@
             }
         }
         /**
-         * Update compiled inference rules for subsequent file processing.
-         *
-         * @param compiledRules - The newly compiled rules.
-         */
-        /**
-         * Update compiled inference rules and optionally the template engine.
+         * Update compiled inference rules, template engine, and custom map lib.
          *
          * @param compiledRules - The newly compiled rules.
          * @param templateEngine - Optional updated template engine.
+         * @param customMapLib - Optional updated custom JsonMap lib functions.
          */
-        updateRules(compiledRules, templateEngine) {
+        updateRules(compiledRules, templateEngine, customMapLib) {
             this.compiledRules = compiledRules;
             if (templateEngine) {
                 this.templateEngine = templateEngine;
             }
+            if (customMapLib !== undefined) {
+                this.config = { ...this.config, customMapLib };
+            }
             this.logger.info({ rules: compiledRules.length }, 'Inference rules updated');
         }
     }
@@ -3020,12 +3133,17 @@
             const compiledRules = this.factories.compileRules(this.config.inferenceRules ?? []);
             const configDir = this.configPath ? node_path.dirname(this.configPath) : '.';
             const templateEngine = await buildTemplateEngine(this.config.inferenceRules ?? [], this.config.templates, this.config.templateHelpers?.paths, configDir);
+            // Load custom JsonMap lib functions
+            const customMapLib = this.config.mapHelpers?.paths?.length && configDir
+                ? await loadCustomMapHelpers(this.config.mapHelpers.paths, configDir)
+                : undefined;
             const processor = this.factories.createDocumentProcessor({
                 metadataDir: this.config.metadataDir ?? '.jeeves-metadata',
                 chunkSize: this.config.embedding.chunkSize,
                 chunkOverlap: this.config.embedding.chunkOverlap,
                 maps: this.config.maps,
                 configDir,
+                customMapLib,
             }, embeddingProvider, vectorStore, compiledRules, logger, templateEngine);
             this.processor = processor;
             this.queue = this.factories.createEventQueue({
@@ -3144,7 +3262,10 @@
                 const compiledRules = this.factories.compileRules(newConfig.inferenceRules ?? []);
                 const reloadConfigDir = node_path.dirname(this.configPath);
                 const newTemplateEngine = await buildTemplateEngine(newConfig.inferenceRules ?? [], newConfig.templates, newConfig.templateHelpers?.paths, reloadConfigDir);
-                processor.updateRules(compiledRules, newTemplateEngine);
+                const newCustomMapLib = newConfig.mapHelpers?.paths?.length && reloadConfigDir
+                    ? await loadCustomMapHelpers(newConfig.mapHelpers.paths, reloadConfigDir)
+                    : undefined;
+                processor.updateRules(compiledRules, newTemplateEngine, newCustomMapLib);
                 logger.info({ configPath: this.configPath, rules: compiledRules.length }, 'Config reloaded');
             }
             catch (error) {
@@ -3191,4 +3312,4 @@
     exports.watchConfigSchema = watchConfigSchema;
     exports.writeMetadata = writeMetadata;
 
-})(this["jeeves-watcher"] = this["jeeves-watcher"] || {}, Fastify, promises, node_path, picomatch, radash, node_crypto, node_fs, ignore, Handlebars, dayjs, hastUtilToMdast, mdastUtilFromAdf, mdastUtilToMarkdown, rehypeParse, unified, chokidar, cosmiconfig, zod, jsonmap, googleGenai, pino, uuid, cheerio, yaml, mammoth, Ajv, addFormats, textsplitters, jsClientRest);
+})(this["jeeves-watcher"] = this["jeeves-watcher"] || {}, Fastify, promises, node_path, picomatch, radash, node_crypto, node_fs, ignore, jsonmap, Handlebars, dayjs, hastUtilToMdast, mdastUtilFromAdf, mdastUtilToMarkdown, rehypeParse, unified, chokidar, cosmiconfig, zod, googleGenai, pino, uuid, cheerio, yaml, mammoth, Ajv, addFormats, textsplitters, jsClientRest);