@chenguangyao/devflow-kit 0.1.43

package/src/providers/drivers/kb-weknora.js

@@ -0,0 +1,472 @@
+'use strict';
+const fs = require('fs/promises');
+const path = require('path');
+const { Provider } = require('../base.js');
+
+/**
+ * WeKnora kb provider — **config-driven**.
+ *
+ * Why config-driven and not "profile-by-name"?
+ *   Different WeKnora deployments (and WeKnora-compatible backends) expose
+ *   slightly different REST contracts. Rather than shipping named presets
+ *   that bake in deployment-specific assumptions (endpoint paths, auth
+ *   headers, response shapes), the driver ships **one generic minimal
+ *   default** and lets the operator supply the full contract via
+ *   `providers.json`. Deployment-specific snippets live outside the package
+ *   (e.g. a local `.internal/providers-*.sample.json`) and are applied by
+ *   hand or via `devflow provider add --from <file>`.
+ *
+ * Config schema (providers.json config{}):
+ *
+ *   {
+ *     // --- required ---
+ *     "endpoint":    "https://your-weknora-host",   // no trailing slash
+ *     "kb_id":       "kb-00000001",                 // knowledge base ID from WeKnora UI
+ *     "token":       "sk-xxxxx",                    // API Key from account settings page
+ *
+ *     // --- optional core ---
+ *     "tag_id":      "",          // default tag_id for uploads
+ *     "idempotent":  true,        // recommended: findByTitle→update-or-create (avoids duplicates)
+ *     "dryRun":      false,       // or env DEVFLOW_KB_DRY_RUN=1
+ *     "timeoutMs":   30000,
+ *
+ *     // --- contract overrides (merged into built-in WeKnora defaults) ---
+ *     // Usually NOT needed for standard WeKnora deployments.
+ *     // Only set if your instance has custom paths or auth.
+ *     "auth":        { "header": "X-API-Key", "scheme": "" },
+ *     "paths":       { "search": { "method": "POST", "path": "/api/v1/knowledge-search" } },
+ *     "body":        { "search": { "query": "{query}", "knowledge_base_id": "{kb_id}" } },
+ *     "responsePath":{ "createId": "data.id", "searchItems": "data",
+ *                      "itemId": "knowledge_id", "itemTitle": "knowledge_title",
+ *                      "itemScore": "score", "itemSnippet": "content" }
+ *   }
+ *
+ * Token resolution order:
+ *   1. config.token / config.api_key  (supports "${env:VAR}" via caller's config loader)
+ *   2. env WEKNORA_TOKEN
+ *   3. env WEKNORA_API_KEY
+ *
+ * Templating:
+ *   Both URL and body strings may contain `{var}` or `{var?}` (optional, dropped if
+ *   unset). Vars available during calls: kb_id, title, content, id, tag_id,
+ *   query, k, category, filename, ids.
+ *
+ * Provider contract:
+ *   validate()                  -> { ok, reason?, needsLogin? }
+ *   upload(file, meta)          -> { uuid, url, action }
+ *   update(uuid, file, meta)
+ *   delete(uuid)
+ *   search(query, opts={ k })   -> [{ title, uuid, url, score, snippet }]
+ */
+
+/**
+ * Built-in contract aligned with the actual WeKnora REST API
+ * (https://github.com/Tencent/WeKnora/blob/main/docs/api/knowledge.md).
+ *
+ * Auth:     X-API-Key header (no Bearer scheme)
+ * Create:   POST /api/v1/knowledge-bases/{kb_id}/knowledge/manual  {title,content,tag_id?}
+ * Update:   PUT  /api/v1/knowledge/manual/{id}                     {title?,content?}
+ * Delete:   DELETE /api/v1/knowledge/{id}
+ * Search:   POST /api/v1/knowledge-search  {query, knowledge_base_id}  → chunks with scores
+ * FindByTitle (idempotent): GET /api/v1/knowledge/search?keyword={title}&limit=5
+ */
+const DEFAULT_CONTRACT = Object.freeze({
+  auth: { header: 'X-API-Key', scheme: '' },  // no scheme prefix — raw token
+  paths: {
+    create: { method: 'POST',   path: '/api/v1/knowledge-bases/{kb_id}/knowledge/manual' },
+    update: { method: 'PUT',    path: '/api/v1/knowledge/manual/{id}' },
+    delete: { method: 'DELETE', path: '/api/v1/knowledge/{id}' },
+    // Vector semantic search (returns ranked chunks)
+    search: { method: 'POST',   path: '/api/v1/knowledge-search' },
+    // Metadata / keyword filter (used by idempotent findByTitle)
+    findByTitle: { method: 'GET', path: '/api/v1/knowledge/search?keyword={title}&limit=5' },
+  },
+  body: {
+    create: { title: '{title}', content: '{content}', tag_id: '{tag_id?}' },
+    update: { title: '{title}', content: '{content}' },
+    // search is POST with JSON body
+    search: { query: '{query}', knowledge_base_id: '{kb_id}' },
+  },
+  responsePath: {
+    createId:    'data.id',    // { data: { id, ... }, success: true }
+    searchItems: 'data',       // { data: [ { knowledge_id, content, score, ... } ] }
+    listItems:   'data',       // GET knowledge list: { data: [...], success: true }
+    itemId:      'knowledge_id', // chunk item field for knowledge ID
+    itemTitle:   'knowledge_title',
+    itemUrl:     'knowledge_source',  // source URL when type=url
+    itemScore:   'score',
+    itemSnippet: 'content',
+  },
+  idempotent: false,
+});
+
+class WeKnoraKbProvider extends Provider {
+  constructor({ name, type, driver, config }) {
+    super({ name, type, driver, config });
+
+    const cfg = config || {};
+    this.contract = {
+      auth:         { ...DEFAULT_CONTRACT.auth,         ...(cfg.auth || {}) },
+      paths:        { ...DEFAULT_CONTRACT.paths,        ...(cfg.paths || {}) },
+      body:         mergeBodyMap(DEFAULT_CONTRACT.body, cfg.body),
+      responsePath: { ...DEFAULT_CONTRACT.responsePath, ...(cfg.responsePath || {}) },
+      idempotent:   cfg.idempotent !== undefined ? !!cfg.idempotent : !!DEFAULT_CONTRACT.idempotent,
+    };
+
+    this.endpoint = (cfg.endpoint || '').replace(/\/+$/, '');
+    this.kbId = cfg.kb_id || cfg.knowledge_base_id || '';
+    this.token = cfg.token
+      || cfg.api_key
+      || process.env.WEKNORA_TOKEN
+      || process.env.WEKNORA_API_KEY
+      || '';
+    this.tagId = cfg.tag_id || '';
+    this.dryRun = !!cfg.dryRun || process.env.DEVFLOW_KB_DRY_RUN === '1';
+    this.timeout = Number(cfg.timeoutMs || 30000);
+  }
+
+  async validate() {
+    if (this.dryRun) return { ok: true };
+    if (!this.endpoint) return { ok: false, reason: 'missing endpoint' };
+    if (!this.kbId) return { ok: false, reason: 'missing kb_id (or knowledge_base_id)' };
+    if (!this.token) return { ok: false, reason: 'missing token (set WEKNORA_TOKEN or config.token)', needsLogin: true };
+    return { ok: true };
+  }
+
+  async upload(file, meta = {}) {
+    const { title, content } = await readMarkdownFile(file);
+    const tagId = meta.tag_id || this.tagId || '';
+    const category = meta.category || '';
+    const filename = meta.filename || path.basename(file);
+
+    if (this.contract.idempotent && this.contract.paths.findByTitle) {
+      const existing = await this._findByTitle(title);
+      if (existing) {
+        const id = getPath(existing, this.contract.responsePath.itemId) || existing.id;
+        await this._call('update', { id, title, content, tag_id: tagId, category, filename });
+        // Re-publish after updating so it re-enters embedding queue
+        await this._reparseById(id);
+        return { uuid: id, url: existing.url || null, action: 'updated' };
+      }
+    }
+    const res = await this._call('create', { title, content, tag_id: tagId, category, filename });
+    // WeKnora: { data: { id, ... }, success: true }
+    const id = getPath(res, this.contract.responsePath.createId)
+            || getPath(res, 'data.id')
+            || (res && res.uuid)
+            || null;
+    const url = getPath(res, this.contract.responsePath.itemUrl) || null;
+
+    // WeKnora manual items are created as drafts (enable_status: disabled).
+    // POST /knowledge/:id/reparse publishes them into the search index.
+    if (id) await this._reparseById(id);
+
+    // Ensure tag association is applied. Some WeKnora builds ignore tag_id in the
+    // create body, so we also call the batch-update-tags endpoint as a guarantee.
+    if (id && tagId) {
+      try { await this.batchUpdateTags({ [id]: tagId }); } catch { /* non-fatal */ }
+    }
+
+    return { uuid: id, url, action: 'created' };
+  }
+
+  async update(uuid, file, meta = {}) {
+    const { title, content } = await readMarkdownFile(file);
+    const tagId = meta.tag_id || this.tagId || '';
+    await this._call('update', { id: uuid, title, content, tag_id: tagId, category: meta.category, filename: meta.filename });
+    // Re-publish so updated content enters the embedding queue
+    await this._reparseById(uuid);
+    // Ensure tag is applied (in case update body didn't carry tag_id through)
+    if (tagId) {
+      try { await this.batchUpdateTags({ [uuid]: tagId }); } catch { /* non-fatal */ }
+    }
+  }
+
+  async delete(uuid) {
+    if (!this.contract.paths.delete) {
+      log('delete-skipped', { uuid, reason: 'contract has no delete endpoint' });
+      return;
+    }
+    await this._call('delete', { id: uuid });
+  }
+
+  /**
+   * Trigger async re-parse (re-embedding) for a knowledge item.
+   * This also transitions enable_status to "enabled" (i.e. "发布入库").
+   * WeKnora: POST /api/v1/knowledge/:id/reparse
+   */
+  async reparse(uuid) {
+    return this._reparseById(uuid);
+  }
+
+  async _reparseById(uuid) {
+    if (!uuid) return null;
+    if (this.dryRun) { log('reparse', { uuid }); return { status: 'dry_run' }; }
+    const url = `${this.endpoint}/api/v1/knowledge/${encodeURIComponent(uuid)}/reparse`;
+    const res = await this._fetch('POST', url, null);
+    return { status: 'reparsing', data: res && res.data };
+  }
+
+  /**
+   * Get parse_status for a knowledge item.
+   * WeKnora: GET /api/v1/knowledge/:id
+   */
+  async getStatus(uuid) {
+    if (this.dryRun) return { parse_status: 'completed', enable_status: 'enabled' };
+    const url = `${this.endpoint}/api/v1/knowledge/${encodeURIComponent(uuid)}`;
+    const res = await this._fetch('GET', url, null);
+    const d = (res && res.data) || res || {};
+    return { parse_status: d.parse_status, enable_status: d.enable_status, title: d.title };
+  }
+
+  async search(query, opts = {}) {
+    const res = await this._call('search', {
+      query,
+      kb_id: this.kbId,
+      tag_id: opts.tag_id || this.tagId || '',
+      k: opts.k || 10,
+    });
+    // WeKnora /knowledge-search returns { data: [ chunk... ], success }
+    // where each chunk has: knowledge_id, knowledge_title, content, score, knowledge_source
+    const itemsPath = this.contract.responsePath.searchItems || 'data';
+    const items = getPath(res, itemsPath) || [];
+    const rp = this.contract.responsePath;
+    return items.map((it) => ({
+      title:   getPath(it, rp.itemTitle)   || it.knowledge_title || it.title || '',
+      uuid:    getPath(it, rp.itemId)      || it.knowledge_id    || it.id    || '',
+      url:     getPath(it, rp.itemUrl)     || it.knowledge_source || it.url  || null,
+      score:   getPath(it, rp.itemScore)   || it.score  || 0,
+      snippet: getPath(it, rp.itemSnippet) || it.content || it.snippet || '',
+    }));
+  }
+
+  // ───────── tag management ─────────
+
+  /**
+   * List all tags in the knowledge base.
+   * WeKnora: GET /api/v1/knowledge-bases/:id/tags
+   * Returns array of { id, name, color, sort_order, knowledge_count }
+   */
+  async listTags() {
+    if (this.dryRun) return [];
+    const url = `${this.endpoint}/api/v1/knowledge-bases/${encodeURIComponent(this.kbId)}/tags?page=1&page_size=200`;
+    const res = await this._fetch('GET', url, null);
+    // { data: { data: [...], total }, success }
+    const items = getPath(res, 'data.data') || getPath(res, 'data') || [];
+    return Array.isArray(items) ? items : [];
+  }
+
+  /**
+   * Create a new tag in the knowledge base.
+   * WeKnora: POST /api/v1/knowledge-bases/:id/tags
+   * Returns the created tag object { id, name, color, sort_order }
+   */
+  async createTag(name, color = '#1890ff', sortOrder = 99) {
+    if (this.dryRun) {
+      const stubId = `dryrun-tag-${name.slice(0, 16)}-${Date.now().toString(36)}`;
+      log('createTag', { name, color });
+      return { id: stubId, name, color };
+    }
+    const url = `${this.endpoint}/api/v1/knowledge-bases/${encodeURIComponent(this.kbId)}/tags`;
+    const res = await this._fetch('POST', url, { name, color, sort_order: sortOrder });
+    return getPath(res, 'data') || res;
+  }
+
+  /**
+   * Batch-update tag associations for multiple knowledge items.
+   * Uses WeKnora's PUT /knowledge/tags bulk endpoint.
+   *
+   * @param {Object} updates  { [uuid]: tagId | null }  — null clears the tag
+   * @returns {boolean} true on success
+   */
+  async batchUpdateTags(updates) {
+    if (!updates || !Object.keys(updates).length) return true;
+    if (this.dryRun) { log('batchUpdateTags', updates); return true; }
+    const url = `${this.endpoint}/api/v1/knowledge/tags`;
+    const res = await this._fetch('PUT', url, { updates });
+    return !!(res && res.success !== false);
+  }
+
+  /**
+   * Idempotent ensure-tag: returns existing tag if name matches, otherwise creates it.
+   * Uses an in-memory cache per provider instance to avoid repeated list calls.
+   */
+  async ensureTag(name, color = '#1890ff', sortOrder = 99) {
+    // Initialise cache once per provider instance
+    if (!this._tagCache) {
+      const tags = await this.listTags();
+      this._tagCache = new Map(tags.map((t) => [t.name, t]));
+    }
+    if (this._tagCache.has(name)) return this._tagCache.get(name);
+    const tag = await this.createTag(name, color, sortOrder);
+    if (tag && tag.id) this._tagCache.set(name, tag);
+    return tag;
+  }
+
+  // ───────── internals ─────────
+
+  async _findByTitle(title) {
+    const tpl = this.contract.paths.findByTitle;
+    if (!tpl) return null;
+    if (this.dryRun) { log('findByTitle', { title }); return null; }
+    const pathTmpl = typeof tpl === 'string' ? tpl : tpl.path;
+    const url = this.endpoint + template(pathTmpl, { kb_id: this.kbId, title });
+    const res = await this._fetch('GET', url, null);
+    // WeKnora GET /knowledge/search returns { data: { data: [...], has_more }, success }
+    // Fall back to flat data array for other deployments.
+    const nested = getPath(res, 'data.data');
+    const flat   = getPath(res, this.contract.responsePath.listItems);
+    const items  = Array.isArray(nested) ? nested : (Array.isArray(flat) ? flat : []);
+    return items.find((it) => it && (it.title === title)) || null;
+  }
+
+  async _call(opKey, ctx) {
+    const endp = this.contract.paths[opKey];
+    if (!endp) throw new Error(`weknora contract has no endpoint for op: ${opKey}`);
+    const { method, pathTmpl } = normaliseEndpoint(endp);
+    const url = this.endpoint + template(pathTmpl, { kb_id: this.kbId, ...ctx });
+    let body = null;
+    if (method !== 'GET' && method !== 'DELETE' && method !== 'HEAD') {
+      const bodyTmpl = this.contract.body[opKey];
+      if (bodyTmpl !== undefined) body = fillBody(bodyTmpl, { kb_id: this.kbId, ...ctx });
+    }
+    if (this.dryRun) {
+      log(opKey, { method, url, body });
+      return stubResponse(opKey, ctx);
+    }
+    return this._fetch(method, url, body);
+  }
+
+  async _fetch(method, url, body) {
+    const headers = { 'accept': 'application/json' };
+    if (body !== null && body !== undefined) headers['content-type'] = 'application/json';
+    const auth = this.contract.auth;
+    if (auth && auth.header && this.token) {
+      // WeKnora uses raw API key without Bearer prefix; other deployments may use scheme
+      headers[auth.header] = (auth.scheme && auth.scheme.trim())
+        ? `${auth.scheme} ${this.token}`
+        : this.token;
+    }
+
+    const ctl = new AbortController();
+    const timer = setTimeout(() => ctl.abort(), this.timeout);
+    let res;
+    try {
+      res = await fetch(url, {
+        method,
+        headers,
+        body: body != null ? JSON.stringify(body) : undefined,
+        signal: ctl.signal,
+      });
+    } finally {
+      clearTimeout(timer);
+    }
+    if (!res.ok) {
+      const text = await res.text().catch(() => '');
+      const err = new Error(`${method} ${url} -> ${res.status} ${res.statusText}\n${text.slice(0, 400)}`);
+      err.status = res.status;
+      if (res.status === 401 || res.status === 403) err.code = 'AUTH_EXPIRED';
+      throw err;
+    }
+    const ct = res.headers.get('content-type') || '';
+    if (ct.includes('json')) return res.json();
+    const text = await res.text();
+    return text ? { raw: text } : null;
+  }
+}
+
+// ─────────── helpers ───────────
+
+/** Read a markdown file, extract title from frontmatter title: or first `# heading`, fall back to filename. */
+async function readMarkdownFile(filePath) {
+  const content = await fs.readFile(filePath, 'utf8');
+  let title = '';
+  const fm = /^---\s*\n([\s\S]*?)\n---/.exec(content);
+  if (fm) {
+    const m = /^title:\s*"?([^"\n]+)"?/m.exec(fm[1]);
+    if (m) title = m[1].trim();
+  }
+  if (!title) {
+    const m = /^#\s+(.+)$/m.exec(content);
+    if (m) title = m[1].trim();
+  }
+  if (!title) title = path.basename(filePath, path.extname(filePath));
+  return { title, content };
+}
+
+/** Normalise paths[op] — accepts string "/x/y" (POST default) or { method, path }. */
+function normaliseEndpoint(entry) {
+  if (typeof entry === 'string') {
+    return { method: 'POST', pathTmpl: entry };
+  }
+  return { method: (entry.method || 'GET').toUpperCase(), pathTmpl: entry.path };
+}
+
+/** URL template: replaces {var} and {var?} with encoded value, leaves other text untouched. */
+function template(str, ctx) {
+  return String(str).replace(/\{(\w+)(\?)?\}/g, (_, k) => {
+    const v = ctx[k];
+    if (v === undefined || v === null || v === '') return '';
+    return encodeURIComponent(String(v));
+  });
+}
+
+/** Body template: recursively resolve {var} / {var?} against ctx. Drop keys with optional+missing values. */
+function fillBody(tpl, ctx) {
+  if (tpl === null || tpl === undefined) return tpl;
+  if (Array.isArray(tpl)) return tpl.map((x) => fillBody(x, ctx)).filter((x) => x !== undefined);
+  if (typeof tpl === 'object') {
+    const out = {};
+    for (const [k, v] of Object.entries(tpl)) {
+      const filled = fillBody(v, ctx);
+      if (filled !== undefined) out[k] = filled;
+    }
+    return out;
+  }
+  if (typeof tpl !== 'string') return tpl;
+  const m = /^\{(\w+)(\?)?\}$/.exec(tpl);
+  if (m) {
+    const v = ctx[m[1]];
+    if (v === undefined || v === null || v === '') return m[2] ? undefined : '';
+    return v;
+  }
+  return tpl.replace(/\{(\w+)(\?)?\}/g, (_, k) => {
+    const v = ctx[k];
+    if (v === undefined || v === null || v === '') return '';
+    return String(v);
+  });
+}
+
+function getPath(obj, dotPath) {
+  if (!obj || !dotPath) return undefined;
+  return dotPath.split('.').reduce((acc, k) => (acc == null ? undefined : acc[k]), obj);
+}
+
+function mergeBodyMap(base = {}, override) {
+  if (!override) return { ...base };
+  const out = { ...base };
+  for (const k of Object.keys(override)) out[k] = override[k];
+  return out;
+}
+
+function stubResponse(opKey, ctx) {
+  const stubId = `dryrun-${opKey}-${(ctx.title || ctx.query || 'x').slice(0, 32)}-${Date.now().toString(36)}`;
+  // WeKnora shapes
+  if (opKey === 'search')  return { data: [], success: true };
+  if (opKey === 'create')  return { data: { id: stubId, type: 'manual', parse_status: 'processing', enable_status: 'disabled' }, success: true };
+  if (opKey === 'update')  return { message: 'Updated successfully', success: true };
+  if (opKey === 'delete')  return { message: 'Deleted successfully', success: true };
+  if (opKey === 'reparse') return { data: { id: ctx.id, parse_status: 'pending' }, success: true };
+  return { success: true };
+}
+
+function log(action, payload) {
+  process.stdout.write(`[kb-weknora dry-run] ${action} ${JSON.stringify(payload)}\n`);
+}
+
+module.exports = {
+  default: WeKnoraKbProvider,
+  WeKnoraKbProvider,
+  DEFAULT_CONTRACT,
+  _internals: { template, fillBody, getPath, readMarkdownFile, normaliseEndpoint },
+};