jsql-neo 6.5.11 → 6.5.13

package/src/cli/plugin.js

@@ -0,0 +1,346 @@
+/**
+ * JSQL Plugin Registry CLI
+ *
+ * 协议文档: /root/.openclaw/workspace/registry-protocol.md
+ * Registry API: https://registry-beta.vexify.top/?action=xxx
+ *
+ * 注意: 当前 registry-beta 实例仅实现了 plugins / search / registry 三个 action，
+ * 尚未实现 info 和 download。本模块优先使用实际 API，缺失能力时给出友好提示。
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+const REGISTRY_BASE = 'https://registry-beta.vexify.top';
+const PLUGIN_DIR = path.join(os.homedir(), '.jsql', 'plugins');
+const MAX_ARCHIVE_BYTES = 50 * 1024 * 1024;
+const MAX_ARCHIVE_ENTRIES = 10000;
+
+// ─── 工具函数 ──────────────────────────────────────────────
+
+/** 插件名只允许字母/数字/点/下划线/连字符，杜绝路径注入 */
+function assertValidPluginName(name) {
+  if (typeof name !== 'string' || !name) {
+    throw new Error('Plugin name must be a non-empty string');
+  }
+  if (name.length > 128) {
+    throw new Error(`Invalid plugin name: too long (${name.length} > 128)`);
+  }
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) || name.includes('..')) {
+    throw new Error(`Invalid plugin name: '${name}'`);
+  }
+  return name;
+}
+
+/** 校验 tar 清单：拒绝绝对路径、`..`、设备/符号链接/硬链接 */
+function assertSafeTarEntries(archivePath, destDir) {
+  const names = execFileSync('tar', ['-tzf', archivePath], {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    maxBuffer: 16 * 1024 * 1024,
+  }).split(/\r?\n/).filter(Boolean);
+  if (names.length > MAX_ARCHIVE_ENTRIES) {
+    throw new Error(`Archive contains too many entries (${names.length} > ${MAX_ARCHIVE_ENTRIES})`);
+  }
+  const destResolved = path.resolve(destDir);
+  for (const entry of names) {
+    if (entry.startsWith('/') || entry.startsWith('\\') || /^[A-Za-z]:[\\/]/.test(entry)) {
+      throw new Error(`Archive contains absolute path: ${entry}`);
+    }
+    if (entry.split(/[\\/]+/).includes('..')) {
+      throw new Error(`Archive contains path traversal: ${entry}`);
+    }
+    const target = path.resolve(destResolved, entry);
+    if (target !== destResolved && !target.startsWith(destResolved + path.sep)) {
+      throw new Error(`Archive entry escapes plugin directory: ${entry}`);
+    }
+  }
+  // 类型检查单独用 -tvzf，避免解析带空格的路径
+  const verbose = execFileSync('tar', ['-tvzf', archivePath], {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    maxBuffer: 16 * 1024 * 1024,
+  });
+  for (const line of verbose.split(/\r?\n/)) {
+    if (!line) continue;
+    const type = line.charAt(0);
+    if (type === 'l' || type === 'h') {
+      throw new Error('Archive contains links, which are not allowed');
+    }
+    if (type !== '-' && type !== 'd') {
+      throw new Error(`Archive contains unsupported entry type '${type}'`);
+    }
+  }
+}
+
+/** 确保插件目录存在 */
+function ensurePluginDir() {
+  if (!fs.existsSync(PLUGIN_DIR)) {
+    fs.mkdirSync(PLUGIN_DIR, { recursive: true });
+  }
+}
+
+/** 从可能含有 PHP Warning 前缀的响应文本中提取 JSON */
+function extractJson(text) {
+  // 找到第一个 '{' 的位置，从那里开始解析
+  const idx = text.indexOf('{');
+  if (idx === -1) return null;
+  try {
+    return JSON.parse(text.slice(idx));
+  } catch {
+    return null;
+  }
+}
+
+/** 发起 registry API 请求，返回解析后的 JSON */
+async function apiRequest(action, params = {}) {
+  const url = new URL(REGISTRY_BASE);
+  url.searchParams.set('action', action);
+  for (const [k, v] of Object.entries(params)) {
+    if (v !== undefined && v !== null) url.searchParams.set(k, v);
+  }
+
+  const res = await fetch(url.toString());
+  const text = await res.text();
+  const body = extractJson(text);
+
+  if (!res.ok || !body || body.status !== 'success') {
+    const msg = body?.error || `HTTP ${res.status}`;
+    throw new Error(`Registry error: ${msg}`);
+  }
+  return body.data;
+}
+
+// ─── 核心操作 ──────────────────────────────────────────────
+
+/** 列出所有插件（支持 category / tag / author / q / sort / order / page / limit） */
+async function listPlugins(opts = {}) {
+  const params = {};
+  if (opts.category) params.category = opts.category;
+  if (opts.tag) params.tag = opts.tag;
+  if (opts.author) params.author = opts.author;
+  if (opts.q) params.q = opts.q;
+  if (opts.sort) params.sort = opts.sort;
+  if (opts.order) params.order = opts.order;
+  if (opts.page) params.page = opts.page;
+  if (opts.limit) params.limit = opts.limit;
+  return apiRequest('plugins', params);
+}
+
+/** 搜索插件（q 参数必填） */
+async function searchPlugins(query, opts = {}) {
+  if (!query) throw new Error('Search query is required');
+  const params = { q: query };
+  if (opts.category) params.category = opts.category;
+  if (opts.tag) params.tag = opts.tag;
+  if (opts.sort) params.sort = opts.sort;
+  return apiRequest('search', params);
+}
+
+/** 获取单个插件信息（实际 API 未实现 info action，通过 plugins 列表过滤） */
+async function getPluginInfo(name) {
+  const plugins = await listPlugins({ q: name, limit: 100 });
+  const plugin = plugins.find(p => p.name === name);
+  if (!plugin) throw new Error(`Plugin '${name}' not found in registry`);
+  return plugin;
+}
+
+/**
+ * 带重试的 fetch（P1 修复：install 网络间歇性失败 30-40%）
+ * 3 次重试 + 指数退避（500ms, 1s, 2s）
+ */
+async function fetchWithRetry(url, retries = 3, baseDelay = 500) {
+  for (let i = 0; i < retries; i++) {
+    try {
+      const res = await fetch(url.toString());
+      if (res.ok) return res;
+      // 5xx / 429 可重试
+      if ((res.status >= 500 || res.status === 429) && i < retries - 1) {
+        const delay = baseDelay * Math.pow(2, i);
+        await new Promise(r => setTimeout(r, delay));
+        continue;
+      }
+      return res;
+    } catch (e) {
+      if (i < retries - 1) {
+        const delay = baseDelay * Math.pow(2, i);
+        await new Promise(r => setTimeout(r, delay));
+        continue;
+      }
+      throw e;
+    }
+  }
+}
+
+/** 安装插件：下载 tgz 并解压到 ~/.jsql/plugins/<name>/ */
+async function installPlugin(name) {
+  assertValidPluginName(name);
+  ensurePluginDir();
+
+  // 检查是否已安装
+  const pluginDir = path.join(PLUGIN_DIR, name);
+  const manifestPath = path.join(pluginDir, 'manifest.json');
+  if (fs.existsSync(manifestPath)) {
+    const existing = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+    throw new Error(`Plugin '${name}' is already installed (v${existing.version})`);
+  }
+
+  // 下载插件包（带重试）
+  const url = new URL(REGISTRY_BASE);
+  url.searchParams.set('action', 'download');
+  url.searchParams.set('name', name);
+
+  const res = await fetchWithRetry(url);
+  const ct = res.headers.get('content-type') || '';
+  const buf = Buffer.from(await res.arrayBuffer());
+
+  if (buf.length > MAX_ARCHIVE_BYTES) {
+    throw new Error(`Plugin archive too large (${buf.length} > ${MAX_ARCHIVE_BYTES} bytes)`);
+  }
+
+  // 检查 Content-Type 或 magic bytes
+  if (!ct.includes('gzip') && buf[0] !== 0x1f) {
+    // 返回了错误 JSON，不是 tgz
+    const text = buf.toString('utf8');
+    const body = extractJson(text);
+    throw new Error(body?.error || `Download failed (HTTP ${res.status})`);
+  }
+
+  const archivePath = path.join(PLUGIN_DIR, `${name}.tgz`);
+  fs.writeFileSync(archivePath, buf, { mode: 0o600 });
+
+  // 解压 tgz：先校验清单，再用 execFile 避免 shell 注入
+  fs.mkdirSync(pluginDir, { recursive: true });
+  try {
+    assertSafeTarEntries(archivePath, pluginDir);
+    execFileSync('tar', ['-xzf', archivePath, '-C', pluginDir], {
+      stdio: 'pipe',
+    });
+  } catch (e) {
+    fs.rmSync(pluginDir, { recursive: true, force: true });
+    fs.rmSync(archivePath, { force: true });
+    throw new Error(`Failed to extract plugin archive: ${e.message}`);
+  }
+
+  // 读取 manifest（兼容两种路径：根级 + plugins/ 子目录）
+  let manifestFile = path.join(pluginDir, 'manifest.json');
+  if (!fs.existsSync(manifestFile)) {
+    manifestFile = path.join(pluginDir, 'plugins', 'manifest.json');
+  }
+  let manifest;
+  if (fs.existsSync(manifestFile)) {
+    manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8'));
+  } else {
+    // 创建默认 manifest
+    manifest = { name, version: '0.0.0', installedAt: new Date().toISOString() };
+    fs.writeFileSync(path.join(pluginDir, 'manifest.json'), JSON.stringify(manifest, null, 2));
+  }
+
+  // 清理压缩包
+  fs.rmSync(archivePath, { force: true });
+
+  return manifest;
+}
+
+/**
+ * 卸载插件
+ * P0 安全修复：校验 name 不含路径遍历序列，使用 basename 隔离，
+ * 防止 `jsql plugin uninstall '../../etc/something'` 删除任意目录。
+ */
+function uninstallPlugin(name) {
+  if (typeof name !== 'string' || !name) {
+    throw new Error('Plugin name must be a non-empty string');
+  }
+  // 拒绝包含路径遍历的名称
+  if (name.includes('..') || name.includes('/') || name.includes('\\')) {
+    throw new Error(`Invalid plugin name: '${name}'`);
+  }
+  // 使用 basename 进一步隔离
+  const safeName = path.basename(name);
+  if (safeName !== name) {
+    throw new Error(`Invalid plugin name: '${name}'`);
+  }
+  const pluginDir = path.join(PLUGIN_DIR, safeName);
+  // 二次校验：确保解析后的路径严格在 PLUGIN_DIR 之下
+  const resolved = path.resolve(pluginDir);
+  const pluginDirResolved = path.resolve(PLUGIN_DIR);
+  if (resolved !== pluginDirResolved && !resolved.startsWith(pluginDirResolved + path.sep)) {
+    throw new Error(`Plugin name resolves outside plugin directory: '${name}'`);
+  }
+  if (!fs.existsSync(pluginDir)) {
+    throw new Error(`Plugin '${safeName}' is not installed`);
+  }
+  fs.rmSync(pluginDir, { recursive: true, force: true });
+  return { name: safeName, uninstalled: true };
+}
+
+/** 列出已安装的插件（兼容根级 + plugins/ 子目录两种布局） */
+function listInstalled() {
+  ensurePluginDir();
+  const entries = fs.readdirSync(PLUGIN_DIR, { withFileTypes: true });
+  const plugins = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    // 兼容两种路径：根级 manifest.json + plugins/ 子目录 manifest.json
+    const manifestPaths = [
+      path.join(PLUGIN_DIR, entry.name, 'manifest.json'),
+      path.join(PLUGIN_DIR, entry.name, 'plugins', 'manifest.json'),
+    ];
+    for (const manifestPath of manifestPaths) {
+      if (fs.existsSync(manifestPath)) {
+        try {
+          plugins.push(JSON.parse(fs.readFileSync(manifestPath, 'utf8')));
+        } catch { /* 跳过损坏的 manifest */ }
+        break;
+      }
+    }
+  }
+  return plugins;
+}
+
+// ─── 格式化输出 ────────────────────────────────────────────
+
+/** 格式化插件列表为表格 */
+function formatPluginList(plugins) {
+  if (!plugins.length) return 'No plugins found.';
+  const rows = plugins.map(p =>
+    `${p.name.padEnd(24)} ${(p.version || '?').padEnd(10)} ${(p.description || '').slice(0, 50)}`
+  );
+  return rows.join('\n');
+}
+
+/** 格式化插件详情 */
+function formatPluginInfo(p) {
+  const lines = [
+    `Name:        ${p.name}`,
+    `Version:     ${p.version}`,
+    `Author:      ${p.author}`,
+    `License:     ${p.license}`,
+    `Category:    ${p.category}`,
+    `Description: ${p.description}`,
+    `Tags:        ${(p.tags || []).join(', ')}`,
+    `Downloads:   ${p.downloads || 0}`,
+  ];
+  if (p.jsql) {
+    lines.push(`JSQL:        ${p.jsql.minVersion}+${p.jsql.maxVersion ? ' – ' + p.jsql.maxVersion : ''}`);
+  }
+  return lines.join('\n');
+}
+
+module.exports = {
+  PLUGIN_DIR,
+  ensurePluginDir,
+  assertValidPluginName,
+  assertSafeTarEntries,
+  listPlugins,
+  searchPlugins,
+  getPluginInfo,
+  installPlugin,
+  uninstallPlugin,
+  listInstalled,
+  formatPluginList,
+  formatPluginInfo,
+  fetchWithRetry,
+};

package/test/plugin-xql.test.js

@@ -0,0 +1,170 @@
+/*
+ * xql 插件测试：把 JSQL 表加载成 XQL 集合，写入自动失效重建。
+ *
+ * 验证点：
+ *   1. 标准 SQL 与管道式语法都能查询 JSQL 表
+ *   2. 索引（hash / range）在查询前生效且结果不变
+ *   3. 插入 / 更新 / 删除后缓存失效，查询反映最新数据
+ *   4. 便捷 API（load / refresh / tables / collection / clear / index）
+ *   5. 未加载且 autoLoad=false 时报错
+ *   6. native 引擎（可用时）同样工作
+ */
+const path = require('path');
+const ROOT = path.join(__dirname, '..');
+const { Database } = require(path.join(ROOT, 'index.js'));
+const plugins = require(path.join(ROOT, 'lib/plugins'));
+const xqlPlugin = plugins.xql;
+
+let pass = 0, fail = 0;
+function ok(name, cond, extra) {
+  if (cond) { pass++; console.log('[OK]  ', name); }
+  else { fail++; console.log('[FAIL]', name, extra !== undefined ? '-> ' + JSON.stringify(extra).slice(0, 140) : ''); }
+}
+const eq = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+const USER_SCHEMA = {
+  id: { type: 'integer' },
+  name: { type: 'string' },
+  age: { type: 'integer' },
+  score: { type: 'integer' },
+};
+
+async function makeDb(Engine, opts) {
+  const db = new Engine(':memory:', { modules: false });
+  await db.start();
+  db.use(xqlPlugin(opts || {}));
+  await db.createTable('users', USER_SCHEMA);
+  await db.insert('users', [
+    { id: 1, name: 'ann', age: 25, score: 90 },
+    { id: 2, name: 'bob', age: 17, score: 70 },
+    { id: 3, name: 'cat', age: 34, score: 55 },
+    { id: 4, name: 'dan', age: 17, score: null },
+  ]);
+  return db;
+}
+
+async function testEngine(Engine, label) {
+  console.log(`\n===== ${label} =====`);
+
+  const db = await makeDb(Engine);
+
+  console.log('\n--- 查询（标准 SQL）---');
+  const rows = await db.xql('SELECT name FROM users WHERE age > 18 ORDER BY name');
+  ok('age > 18 排序返回', eq(rows, [{ name: 'ann' }, { name: 'cat' }]), rows);
+
+  console.log('\n--- 查询（管道式）---');
+  const pipe = await db.xql('FROM users |> WHERE age > 18 |> SELECT name |> ORDER BY name');
+  ok('管道式与标准 SQL 等价', eq(pipe, rows), pipe);
+
+  console.log('\n--- 索引 ---');
+  db.xqlIndex('users', 'id');
+  db.xqlIndex('users', 'score', 'range');
+  const coll = db.xqlCollection('users');
+  ok('集合已建 id hash 索引', !!(coll && coll._indexes.has('id')));
+  ok('集合已建 score range 索引', !!(coll && coll._indexes.has('score')));
+  const eqById = await db.xql('SELECT id FROM users WHERE id = 2');
+  ok('hash 索引点查', eq(eqById, [{ id: 2 }]), eqById);
+  const rng = await db.xql('SELECT id FROM users WHERE score >= 70 ORDER BY id');
+  ok('range 索引范围查', eq(rng, [{ id: 1 }, { id: 2 }]), rng);
+  ok('index 记录在 tables 里', db.xqlTables().includes('users'));
+
+  console.log('\n--- 写入失效 ---');
+  await db.insert('users', [{ id: 5, name: 'eve', age: 40, score: 88 }]);
+  const afterIns = await db.xql('SELECT name FROM users WHERE id = 5');
+  ok('插入后立即可见（缓存重建）', eq(afterIns, [{ name: 'eve' }]), afterIns);
+
+  await db.updateById('users', 2, { age: 99 });
+  const afterUpd = await db.xql('SELECT name FROM users WHERE age > 98');
+  ok('更新后立即可见', eq(afterUpd, [{ name: 'bob' }]), afterUpd);
+
+  await db.removeById('users', 1);
+  const afterDel = await db.xql('SELECT name FROM users WHERE id = 1');
+  ok('删除后不可见', eq(afterDel, []), afterDel);
+
+  console.log('\n--- 便捷 API ---');
+  const t = await db.xqlLoad('users');
+  ok('load 返回集合', t && t.size === 4, t && t.size);
+  const r = await db.xqlRefresh('users');
+  ok('refresh 重建', r && r.size === 4);
+  ok('tables() 有 users', db.xqlTables().includes('users'));
+  ok('collection() 取到', db.xqlCollection('users') === r);
+  db.xqlClear('users');
+  ok('clear 后 collection 为空', db.xqlCollection('users') === null);
+  ok('clear 后 tables 为空', !db.xqlTables().includes('users'));
+  const auto = await db.xql('SELECT name FROM users WHERE age > 18 ORDER BY name');
+  ok('clear 后再查自动重载', eq(auto, [{ name: 'bob' }, { name: 'cat' }, { name: 'eve' }]), auto);
+
+  await db.stop();
+}
+
+async function testAutoLoadOff() {
+  console.log('\n===== autoLoad=false =====');
+  const db = new Database(':memory:', { modules: false });
+  await db.start();
+  db.use(xqlPlugin({ autoLoad: false }));
+  await db.createTable('users', USER_SCHEMA);
+  await db.insert('users', [{ id: 1, name: 'ann', age: 25, score: 90 }]);
+  let threw = false;
+  try { await db.xql('SELECT name FROM users'); } catch (e) { threw = /未加载/.test(e.message); }
+  ok('autoLoad=false 未加载报错', threw);
+  await db.xqlLoad('users');
+  const r = await db.xql('SELECT name FROM users');
+  ok('显式 load 后可查', eq(r, [{ name: 'ann' }]), r);
+  await db.stop();
+}
+
+async function testTableName() {
+  console.log('\n===== 显式数据源 opts.table =====');
+  const db = new Database(':memory:', { modules: false });
+  await db.start();
+  db.use(xqlPlugin());
+  await db.createTable('users', USER_SCHEMA);
+  await db.insert('users', [{ id: 1, name: 'ann', age: 25, score: 90 }]);
+  const r = await db.xql('SELECT name FROM users WHERE age > 18', { table: 'users' });
+  ok('opts.table 指定源', eq(r, [{ name: 'ann' }]), r);
+  await db.stop();
+}
+
+async function testInstalledPluginAutoLoad() {
+  console.log('\n===== 注册表插件自动加载（entry 仓库相对路径 + 根级文件）=====');
+  const os = require('os');
+  const fs = require('fs');
+  const home = os.homedir();
+  const name = '_xqltest_registry';
+  const dir = path.join(home, '.jsql', 'plugins', name);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(path.join(dir, 'index.js'),
+    "module.exports = { name: '" + name + "', install(engine) { engine.__xqltest = true; } };\n");
+  fs.writeFileSync(path.join(dir, 'manifest.json'),
+    JSON.stringify({ name, version: '1.0.0', entry: 'plugins/' + name + '/index.js', files: [] }));
+  try {
+    const db = new Database(':memory:', { modules: true });
+    await db.start();
+    ok('仓库相对 entry + 根级 index.js 能自动加载', db.__xqltest === true);
+    await db.stop();
+  } finally {
+    try { fs.rmSync(path.join(home, '.jsql', 'plugins', name), { recursive: true, force: true }); } catch { /* ignore */ }
+  }
+}
+
+(async () => {
+  await testEngine(Database, 'Database (Pure JS)');
+  await testAutoLoadOff();
+  await testTableName();
+  await testInstalledPluginAutoLoad();
+
+  // native 引擎（Linux 上可用；Android 等平台无二进制时跳过）
+  try {
+    const Native = require(path.join(ROOT, 'lib/native_client')).JSQL;
+    if (typeof Native === 'function') {
+      await testEngine(Native, 'NativeJSQL (Rust)');
+    } else {
+      console.log('\n[native] JSQL export not a constructor, skip');
+    }
+  } catch (e) {
+    console.log('\n[native] 不可用，跳过:', e.message.slice(0, 80));
+  }
+
+  console.log(`\n=== pass ${pass}  fail ${fail} ===`);
+  process.exit(fail ? 1 : 0);
+})();