@devilsdev/rag-pipeline-utils 2.4.4 → 2.5.0

package/src/ecosystem/plugin-hub.js

@@ -145,6 +145,23 @@ class PluginHub extends EventEmitter {
   /**
    * Install plugin from the hub
    */
+  /**
+   * Download a plugin from the registry and install it into a project
+   * (see ecosystem/plugin-installer.js): sha256 verified against the
+   * registry, static security scan (high-risk findings block unless
+   * allowRisky), stage contract check, files kept inside
+   * <project>/.rag/plugins, and `plugins.<stage>.<name>` added to the config.
+   *
+   * @param {string} pluginId - Registry plugin id
+   * @param {string} [version='latest']
+   * @param {object} [_options]
+   * @param {string} [_options.configPath='.ragrc.json'] - Config to register the plugin in
+   * @param {boolean} [_options.requireCertified] - Refuse plugins the registry marks uncertified
+   * @param {string} [_options.stage] - Overrides package.json ragPipeline.stage
+   * @param {string} [_options.name] - Overrides package.json ragPipeline.name
+   * @param {boolean} [_options.force] - Replace an installed plugin
+   * @param {boolean} [_options.allowRisky] - Install despite high-risk findings
+   */
   async installPlugin(pluginId, version = 'latest', _options = {}) {
     const installId = crypto.randomUUID();
     const startedAt = Date.now();
@@ -152,69 +169,43 @@ class PluginHub extends EventEmitter {
     try {
       this.emit('install_start', { pluginId, version, installId });
 
-      // Get plugin info and verify
-      const pluginInfo = await this.getPluginInfo(pluginId);
-      if (!pluginInfo) throw new Error(`Plugin ${pluginId} not found`);
-
-      // Check certification if required
-      if (_options.requireCertified && !pluginInfo.certified) {
-        throw new Error(`Plugin ${pluginId} is not certified`);
-      }
-
-      // Security scan in sandbox
-      if (_options.securityScan !== false) {
-        this.emit('install_progress', { installId, stage: 'security_scan' });
-        await this.sandbox.scanPlugin(pluginInfo);
+      if (_options.requireCertified) {
+        const info = await this.getPluginInfo(pluginId);
+        if (!info.certified)
+          throw new Error(`Plugin ${pluginId} is not certified`);
       }
 
-      // Download plugin
-      this.emit('install_progress', { installId, stage: 'downloading' });
-      const downloadUrl = await this._getDownloadUrl(pluginId, version);
-      const pluginData = await this._downloadPlugin(downloadUrl);
-
-      // Verify integrity
-      this.emit('install_progress', { installId, stage: 'verifying' });
-      await this._verifyPluginIntegrity(pluginData, pluginInfo.checksums);
-
-      // Install in sandbox first
-      this.emit('install_progress', { installId, stage: 'sandbox_install' });
-      const sandboxResult = await this.sandbox.installPlugin(pluginData, {
-        timeout: _options.sandboxTimeout || 30000,
-        memoryLimit: _options.memoryLimit || '512MB',
-        networkAccess: _options.networkAccess || false,
+      const { installPlugin } = require('./plugin-installer');
+      const result = await installPlugin({
+        source: pluginId,
+        version,
+        hub: this,
+        configPath: _options.configPath || '.ragrc.json',
+        stage: _options.stage,
+        name: _options.name,
+        force: _options.force,
+        allowRisky: _options.allowRisky,
+        onProgress: (stage) =>
+          this.emit('install_progress', { installId, stage }),
       });
 
-      if (!sandboxResult.success) {
-        throw new Error(`Sandbox installation failed: ${sandboxResult.error}`);
-      }
-
-      // Install to main system
-      this.emit('install_progress', { installId, stage: 'installing' });
-      const installPath = await this._installPluginToSystem(
-        pluginData,
-        pluginInfo,
-      );
-
-      // Track installation analytics
       this.analytics.trackInstallation(pluginId, version, {
         success: true,
         installTime: Date.now() - startedAt,
       });
-
       this.emit('install_complete', {
         pluginId,
         version,
         installId,
-        installPath,
-        pluginInfo,
+        installPath: result.path,
       });
 
       return {
         success: true,
         pluginId,
         version,
-        installPath,
-        pluginInfo,
+        installPath: result.path,
+        ...result,
       };
     } catch (error) {
       this.analytics.trackInstallation(pluginId, version, {
@@ -344,46 +335,16 @@ class PluginHub extends EventEmitter {
   /**
    * Get user's installed plugins
    */
-  async getInstalledPlugins() {
-    try {
-      const pluginsDir = path.join(this._config.localCacheDir, 'plugins');
-      const entries = await fs.readdir(pluginsDir, { withFileTypes: true });
-
-      const installed = [];
-
-      for (const entry of entries) {
-        if (entry.isDirectory()) {
-          const metadataPath = path.join(
-            pluginsDir,
-            entry.name,
-            'metadata.json',
-          );
-
-          try {
-            const metadata = JSON.parse(
-              await fs.readFile(metadataPath, 'utf8'),
-            );
-            installed.push({
-              ...metadata,
-              installPath: path.join(pluginsDir, entry.name),
-              lastUsed: await this.analytics.getLastUsed(metadata.id),
-            });
-          } catch {
-            // Skip invalid plugins
-            continue; // eslint-disable-line no-continue
-          }
-        }
-      }
-
-      return installed;
-    } catch {
-      return [];
-    }
+  /**
+   * Plugins registered in a project's config (see plugin-installer
+   * listInstalledPlugins).
+   * @param {string} [configPath='.ragrc.json']
+   */
+  async getInstalledPlugins(configPath = '.ragrc.json') {
+    const { listInstalledPlugins } = require('./plugin-installer');
+    return listInstalledPlugins(configPath);
   }
 
-  // ------------------------------
-  // Private methods
-  // ------------------------------
   async _makeRequest(method, endpoint, params = {}) {
     const doFetch = pickFetch(this._config.fetch);
     if (!doFetch) throw new Error('fetch is not available in this environment');
@@ -540,35 +501,6 @@ class PluginHub extends EventEmitter {
     }
   }
 
-  async _installPluginToSystem(pluginData, pluginInfo) {
-    const installDir = path.join(
-      this._config.localCacheDir,
-      'plugins',
-      pluginInfo.id,
-    );
-
-    // Create installation directory
-    await fs.mkdir(installDir, { recursive: true });
-
-    // Simulated install: write metadata only
-    const metadataPath = path.join(installDir, 'metadata.json');
-    await fs.writeFile(
-      metadataPath,
-      JSON.stringify(
-        {
-          ...pluginInfo,
-          installedAt: new Date().toISOString(),
-          installPath: installDir,
-          size: Buffer.isBuffer(pluginData) ? pluginData.length : 0,
-        },
-        null,
-        2,
-      ),
-    );
-
-    return installDir;
-  }
-
   async _validatePluginForPublish(pluginPath) {
     // Validate plugin structure and metadata
     const packageJsonPath = path.join(pluginPath, 'package.json');
@@ -608,19 +540,12 @@ class PluginHub extends EventEmitter {
     });
   }
 
-  async _getInstalledVersion(pluginId) {
-    try {
-      const metadataPath = path.join(
-        this._config.localCacheDir,
-        'plugins',
-        pluginId,
-        'metadata.json',
-      );
-      const metadata = JSON.parse(await fs.readFile(metadataPath, 'utf8'));
-      return metadata.version;
-    } catch {
-      return null;
-    }
+  async _getInstalledVersion(pluginId, configPath = '.ragrc.json') {
+    const { listInstalledPlugins } = require('./plugin-installer');
+    const match = listInstalledPlugins(configPath).find(
+      (p) => p.name === pluginId,
+    );
+    return match ? match.version : null;
   }
 }
 

package/src/ecosystem/plugin-installer.js

@@ -0,0 +1,298 @@
+'use strict';
+
+/**
+ * Install a plugin into a project and register it in .ragrc.json.
+ *
+ * Sources: a registry plugin id (needs a PluginHub; the download is verified
+ * against the registry's sha256), a local directory, a .tgz / .tar.gz
+ * package, a single .js file, or an npm package ("npm:<spec>").
+ *
+ * Steps: stage the files inside <project>/.rag/plugins, statically scan
+ * them (high-risk findings block the install unless allowRisky), check the
+ * entry module exposes the stage's required methods (in a separate process),
+ * move into place and add `plugins.<stage>.<name>` to the config. Install
+ * scripts are never run. Installed plugins run with your privileges when the
+ * pipeline uses them; the scan informs that decision, it does not sandbox.
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const crypto = require('crypto');
+const { execFile } = require('child_process');
+const { promisify } = require('util');
+const { extractTarGz, isWithin, relativeWithin } = require('./tar-extract');
+const { scanPlugin } = require('./plugin-scanner');
+const { probeContract, STAGE_METHODS } = require('./plugin-probe');
+
+const NAME_PATTERN = /^[a-z0-9][a-z0-9._-]{0,63}$/i;
+
+/** child is strictly inside parent (both resolved paths). */
+function isInside(parent, child) {
+  return (
+    isWithin(parent, child) && path.resolve(parent) !== path.resolve(child)
+  );
+}
+
+/** Run npm without a shell (on Windows via node + npm-cli.js). */
+async function runNpm(args, cwd) {
+  const run = promisify(execFile);
+  if (process.platform === 'win32') {
+    const npmCli = path.join(
+      path.dirname(process.execPath),
+      'node_modules',
+      'npm',
+      'bin',
+      'npm-cli.js',
+    );
+    return run(process.execPath, [npmCli, ...args], { cwd, windowsHide: true });
+  }
+  return run('npm', args, { cwd });
+}
+
+/** If extraction produced a single top-level directory (e.g. "package/"), use it. */
+function packageRoot(dir) {
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  if (entries.length === 1 && entries[0].isDirectory()) {
+    return path.join(dir, entries[0].name);
+  }
+  return dir;
+}
+
+async function stageSource(source, stagingDir, options) {
+  const local = path.resolve(options.cwd || process.cwd(), source);
+
+  if (fs.existsSync(local)) {
+    const stat = fs.statSync(local);
+    if (stat.isDirectory()) {
+      fs.cpSync(local, stagingDir, {
+        recursive: true,
+        // Skip node_modules and .git at any depth (path is relative to local)
+        filter: (src) =>
+          !/(^|\/)(node_modules|\.git)(\/|$)/.test(
+            relativeWithin(local, src) || '',
+          ),
+      });
+      return { kind: 'directory', root: stagingDir };
+    }
+    if (/\.(tgz|tar\.gz)$/i.test(local)) {
+      extractTarGz(fs.readFileSync(local), stagingDir);
+      return { kind: 'tarball', root: packageRoot(stagingDir) };
+    }
+    if (/\.(c?js)$/i.test(local)) {
+      fs.copyFileSync(local, path.join(stagingDir, 'index.js'));
+      return { kind: 'file', root: stagingDir };
+    }
+    throw new Error(
+      `Unsupported plugin file: ${source} (expected a directory, .tgz or .js)`,
+    );
+  }
+
+  if (source.startsWith('npm:')) {
+    const spec = source.slice(4);
+    const packDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rag-npm-'));
+    try {
+      const { stdout } = await runNpm(
+        [
+          'pack',
+          spec,
+          '--silent',
+          '--ignore-scripts',
+          '--pack-destination',
+          packDir,
+        ],
+        packDir,
+      );
+      const tarball = path.join(packDir, stdout.trim().split(/\r?\n/).pop());
+      extractTarGz(fs.readFileSync(tarball), stagingDir);
+      return { kind: 'npm', root: packageRoot(stagingDir) };
+    } finally {
+      fs.rmSync(packDir, { recursive: true, force: true });
+    }
+  }
+
+  if (!options.hub) {
+    throw new Error(
+      `"${source}" is not a local path or npm:<package>, and no plugin registry is configured.`,
+    );
+  }
+  const hub = options.hub;
+  const info = await hub.getPluginInfo(source);
+  const url = await hub._getDownloadUrl(source, options.version || 'latest');
+  const archive = await hub._downloadPlugin(url);
+  await hub._verifyPluginIntegrity(archive, info.checksums); // throws on mismatch
+  extractTarGz(archive, stagingDir);
+  return { kind: 'registry', root: packageRoot(stagingDir), info };
+}
+
+/**
+ * @param {object} params
+ * @param {string} params.source - Registry id, path, .tgz, .js or "npm:<spec>"
+ * @param {string} params.configPath - .ragrc.json to register the plugin in
+ * @param {string} [params.stage] - Overrides package.json "ragPipeline.stage"
+ * @param {string} [params.name] - Overrides package.json "ragPipeline.name"
+ * @param {boolean} [params.force] - Replace an installed plugin with the same stage/name
+ * @param {boolean} [params.allowRisky] - Install despite high-risk scan findings
+ * @param {object} [params.hub] - PluginHub, required for registry ids
+ * @param {string} [params.version] - Registry version (default "latest")
+ * @param {function} [params.onProgress] - Called with a stage label
+ * @returns {Promise<object>} Installation summary
+ */
+async function installPlugin(params) {
+  const progress = params.onProgress || (() => {});
+  const configPath = path.resolve(params.configPath);
+  if (!fs.existsSync(configPath)) {
+    throw new Error(
+      `Config file not found: ${params.configPath}. Run "rag-pipeline init" first.`,
+    );
+  }
+  const configDir = path.dirname(configPath);
+  const pluginsDir = path.join(configDir, '.rag', 'plugins');
+  fs.mkdirSync(pluginsDir, { recursive: true });
+  const stagingDir = path.join(
+    pluginsDir,
+    `.staging-${crypto.randomBytes(6).toString('hex')}`,
+  );
+  fs.mkdirSync(stagingDir);
+
+  try {
+    progress('fetching');
+    const staged = await stageSource(params.source, stagingDir, {
+      hub: params.hub,
+      version: params.version,
+      cwd: configDir,
+    });
+
+    const manifestPath = path.join(staged.root, 'package.json');
+    const manifest = fs.existsSync(manifestPath)
+      ? JSON.parse(fs.readFileSync(manifestPath, 'utf8'))
+      : {};
+    const stage = params.stage || manifest.ragPipeline?.stage;
+    const name = params.name || manifest.ragPipeline?.name;
+    if (!STAGE_METHODS[stage]) {
+      throw new Error(
+        `Unknown or missing stage. Pass --stage (${Object.keys(STAGE_METHODS).join(', ')}) or set "ragPipeline.stage" in package.json.`,
+      );
+    }
+    if (!name || !NAME_PATTERN.test(name)) {
+      throw new Error(
+        'Missing or invalid plugin name. Pass --name (letters, digits, ".", "_", "-") or set "ragPipeline.name" in package.json.',
+      );
+    }
+
+    const entry = path.resolve(staged.root, manifest.main || 'index.js');
+    if (!isInside(staged.root, entry) || !fs.existsSync(entry)) {
+      throw new Error(
+        `Plugin entry file not found: ${manifest.main || 'index.js'}`,
+      );
+    }
+
+    progress('scanning');
+    const scan = scanPlugin(staged.root);
+    if (scan.counts.high > 0 && !params.allowRisky) {
+      const error = new Error(
+        `Security scan found ${scan.counts.high} high-risk finding(s); not installed. Review them and re-run with --allow-risky to install anyway.`,
+      );
+      error.scan = scan;
+      throw error;
+    }
+
+    progress('verifying');
+    const probe = await probeContract(entry, stage);
+    if (!probe.ok) {
+      const reason =
+        probe.error || `missing method(s): ${probe.missing.join(', ')}`;
+      throw new Error(
+        `Plugin does not implement the ${stage} contract: ${reason}`,
+      );
+    }
+
+    progress('installing');
+    const destination = path.join(pluginsDir, `${stage}-${name}`);
+    if (fs.existsSync(destination)) {
+      if (!params.force) {
+        throw new Error(
+          `${stage} plugin "${name}" is already installed. Use --force to replace it.`,
+        );
+      }
+      fs.rmSync(destination, { recursive: true, force: true });
+    }
+    fs.renameSync(staged.root, destination);
+    const installedEntry = path.join(
+      destination,
+      relativeWithin(staged.root, entry),
+    );
+
+    const raw = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    raw.plugins = raw.plugins || {};
+    raw.plugins[stage] = raw.plugins[stage] || {};
+    raw.plugins[stage][name] = `./${relativeWithin(configDir, installedEntry)}`;
+    fs.writeFileSync(configPath, `${JSON.stringify(raw, null, 2)}\n`);
+
+    return {
+      stage,
+      name,
+      source: staged.kind,
+      version: manifest.version || staged.info?.version || null,
+      path: destination,
+      entry: raw.plugins[stage][name],
+      scan,
+      methods: probe.methods,
+      inPipeline: Array.isArray(raw.pipeline)
+        ? raw.pipeline.some((p) => p.stage === stage && p.name === name)
+        : false,
+    };
+  } finally {
+    fs.rmSync(stagingDir, { recursive: true, force: true });
+  }
+}
+
+module.exports = { installPlugin };
+
+/**
+ * Plugins registered under `plugins` in a config, with the installed
+ * package's version when it lives in .rag/plugins.
+ *
+ * @param {string} configPath
+ * @returns {Array<{stage: string, name: string, entry: string, version: string|null, installed: boolean, inPipeline: boolean, exists: boolean}>}
+ */
+function listInstalledPlugins(configPath) {
+  const absolute = path.resolve(configPath);
+  if (!fs.existsSync(absolute)) return [];
+  const config = JSON.parse(fs.readFileSync(absolute, 'utf8'));
+  const configDir = path.dirname(absolute);
+  const pluginsDir = path.join(configDir, '.rag', 'plugins');
+  const pipeline = Array.isArray(config.pipeline) ? config.pipeline : [];
+  const list = [];
+
+  for (const [stage, plugins] of Object.entries(config.plugins || {})) {
+    for (const [name, entry] of Object.entries(plugins || {})) {
+      if (typeof entry !== 'string') continue;
+      const entryPath = path.resolve(configDir, entry);
+      const installedDir = path.join(pluginsDir, `${stage}-${name}`);
+      const installed = isInside(installedDir, entryPath);
+      let version = null;
+      const manifest = path.join(installedDir, 'package.json');
+      if (installed && fs.existsSync(manifest)) {
+        try {
+          version =
+            JSON.parse(fs.readFileSync(manifest, 'utf8')).version || null;
+        } catch {
+          // unreadable manifest: version unknown
+        }
+      }
+      list.push({
+        stage,
+        name,
+        entry,
+        version,
+        installed,
+        exists: fs.existsSync(entryPath),
+        inPipeline: pipeline.some((p) => p.stage === stage && p.name === name),
+      });
+    }
+  }
+  return list;
+}
+
+module.exports.listInstalledPlugins = listInstalledPlugins;

package/src/ecosystem/plugin-probe.js

@@ -0,0 +1,127 @@
+'use strict';
+
+/**
+ * Check that a plugin module exposes the methods its stage requires
+ * (contracts/<stage>-contract.json). Loading a module runs its code, so the
+ * check happens in a separate Node process with a timeout; this keeps the
+ * calling process clean but is NOT a security sandbox.
+ */
+
+const { execFile } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+const STAGE_METHODS = {
+  loader: ['load'],
+  embedder: ['embed'],
+  retriever: ['store', 'retrieve'],
+  llm: ['generate'],
+  reranker: ['rerank'],
+};
+
+// Runs in the child: load the module the way buildComponents() does and
+// report which methods the resulting object has.
+const MARKER = '@@rag-probe@@';
+const PROBE_SCRIPT = `
+const MARKER = '${MARKER}';
+const mod = require(process.argv[1]);
+const exported = mod && mod.default ? mod.default : mod;
+let instance = exported;
+if (typeof exported === 'function') {
+  instance = /^class\\s/.test(Function.prototype.toString.call(exported))
+    ? new exported({})
+    : exported({});
+}
+const methods = [];
+for (let o = instance; o && o !== Object.prototype; o = Object.getPrototypeOf(o)) {
+  for (const key of Object.getOwnPropertyNames(o)) {
+    try { if (key !== 'constructor' && typeof instance[key] === 'function') methods.push(key); } catch {}
+  }
+}
+process.stdout.write(MARKER + JSON.stringify({ type: typeof instance, methods: [...new Set(methods)] }));
+`;
+
+/** The "Error: ..." line of a Node stack trace, else the first line. */
+function firstErrorLine(text) {
+  const lines = String(text)
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+  return (
+    lines.find((line) => /^\w*Error\b/.test(line)) ||
+    lines[0] ||
+    'unknown error'
+  );
+}
+
+/**
+ * @param {string} modulePath - Plugin entry file
+ * @param {string} stage - loader | embedder | retriever | llm | reranker
+ * @param {{timeoutMs?: number}} [options]
+ * @returns {Promise<{ok: boolean, missing: string[], methods: string[], error?: string}>}
+ */
+function probeContract(modulePath, stage, options = {}) {
+  const required = STAGE_METHODS[stage];
+  if (!required) {
+    return Promise.reject(
+      new Error(
+        `Unknown stage "${stage}". Expected: ${Object.keys(STAGE_METHODS).join(', ')}`,
+      ),
+    );
+  }
+  const absolute = path.resolve(modulePath);
+  if (!fs.existsSync(absolute)) {
+    return Promise.resolve({
+      ok: false,
+      missing: required,
+      methods: [],
+      error: `Not found: ${modulePath}`,
+    });
+  }
+
+  return new Promise((resolve) => {
+    execFile(
+      process.execPath,
+      ['-e', PROBE_SCRIPT, absolute],
+      {
+        timeout: options.timeoutMs || 10000,
+        cwd: path.dirname(absolute),
+        windowsHide: true,
+      },
+      (error, stdout, stderr) => {
+        if (error) {
+          const reason = error.killed
+            ? 'timed out while loading'
+            : firstErrorLine(stderr || error.message);
+          resolve({
+            ok: false,
+            missing: required,
+            methods: [],
+            error: `Failed to load: ${reason}`,
+          });
+          return;
+        }
+        let parsed;
+        try {
+          // The module may print while loading; the result follows the marker.
+          parsed = JSON.parse(
+            stdout.slice(stdout.lastIndexOf(MARKER) + MARKER.length),
+          );
+        } catch {
+          resolve({
+            ok: false,
+            missing: required,
+            methods: [],
+            error: 'Module printed unexpected output while loading',
+          });
+          return;
+        }
+        // An embedder may implement embedQuery only for queries; embed is required.
+        const missing = required.filter((m) => !parsed.methods.includes(m));
+        resolve({ ok: missing.length === 0, missing, methods: parsed.methods });
+      },
+    );
+  });
+}
+
+module.exports = { probeContract, STAGE_METHODS };