@chenguangyao/devflow-kit 0.1.43

package/src/providers/drivers/kb-git.js

@@ -0,0 +1,549 @@
+'use strict';
+/**
+ * kb-git provider — knowledge base backed by a standalone GitLab/GitHub Git repo.
+ *
+ * Mirrors arb-workflow-kit's arb_knowledge.py design:
+ *   ensure()        — clone / checkout main / pull
+ *   listTags()      — enumerate .meta.json across all category dirs
+ *   resolveId()     — reverse-lookup local path from knowledge_id
+ *   depositAndCreateMr() — stash → checkout main → pull → branch → pop →
+ *                          write files → update .meta.json → add+commit+push → MR
+ *
+ * config shape (set via `devflow provider setup`, key "kb.git"):
+ * {
+ *   repoUrl:        string  (required) git clone URL
+ *   localPath:      string  (required) local clone path, e.g. ~/.devflow/kb/arb-knowledge
+ *   targetBranch:   string  (optional, default "main") base branch / MR target
+ *   gitlabApiUrl:   string  (optional) https://code.corp.com/api/v4
+ *   gitlabToken:    string  (optional) PRIVATE-TOKEN
+ *   projectId:      string  (optional) "group/project" or numeric ID
+ *   githubToken:    string  (optional) GitHub PAT
+ *   githubRepo:     string  (optional) "owner/repo" (auto-detected from repoUrl)
+ *   mrReviewers:    string  (optional) comma-separated GitLab usernames
+ * }
+ */
+
+const fs      = require('fs/promises');
+const fsSync  = require('fs');
+const path    = require('path');
+const https   = require('https');
+const http    = require('http');
+const cp      = require('child_process');
+const os      = require('os');
+const { Provider } = require('../base.js');
+
+class KbGitProvider extends Provider {
+  constructor({ name, type, driver, config }) {
+    super({ name, type, driver, config });
+    this.repoUrl      = config.repoUrl || '';
+    this.localPath    = expandHome(config.localPath || '');
+    this.targetBranch = config.targetBranch || 'main';
+    this.gitlabApiUrl = (config.gitlabApiUrl || '').replace(/\/+$/, '');
+    this.gitlabToken  = config.gitlabToken || '';
+    this.projectId    = config.projectId || '';
+    this.githubToken  = config.githubToken || '';
+    this.githubRepo   = config.githubRepo || '';
+    this.mrReviewers  = (config.mrReviewers || '').split(',').map((s) => s.trim()).filter(Boolean);
+  }
+
+  async validate() {
+    if (!this.repoUrl)   return { ok: false, reason: 'repoUrl is required' };
+    if (!this.localPath) return { ok: false, reason: 'localPath is required' };
+    try {
+      await this.ensure();
+      return { ok: true };
+    } catch (e) {
+      return { ok: false, reason: e.message };
+    }
+  }
+
+  // ── Knowledge sync interface (devflow knowledge sync) ───────────────────────
+
+  async upload(filePath, { category, filename } = {}) {
+    await this.ensure();
+    const dest = path.join(this.localPath, category || '', filename || path.basename(filePath));
+    await fs.mkdir(path.dirname(dest), { recursive: true });
+    await fs.copyFile(filePath, dest);
+    this._gitCommitPush(`chore(kb): sync ${filename || path.basename(filePath)}`);
+    return { uuid: dest, url: null };
+  }
+
+  async update(uuid, filePath) {
+    await fs.copyFile(filePath, uuid);
+    this._gitCommitPush(`chore(kb): update ${path.basename(uuid)}`);
+  }
+
+  async delete(uuid) {
+    if (fsSync.existsSync(uuid)) await fs.unlink(uuid);
+    this._gitCommitPush(`chore(kb): delete ${path.basename(uuid)}`);
+  }
+
+  async search(q, { k = 8 } = {}) {
+    await this.ensure();
+    const hits = [];
+    const files = await walkMd(this.localPath);
+    const kws = q.toLowerCase().split(/\s+/).filter(Boolean);
+    for (const f of files) {
+      const content = await fs.readFile(f, 'utf8');
+      const lower   = content.toLowerCase();
+      let score = 0;
+      for (const kw of kws) {
+        if (path.basename(f).toLowerCase().includes(kw)) score += 5;
+        const headings = (content.match(/^#{1,2}\s+.+$/gm) || []).join('\n').toLowerCase();
+        if (headings.includes(kw)) score += 3;
+        score += Math.min((lower.match(new RegExp(kw, 'g')) || []).length, 10);
+      }
+      score = +(score / Math.sqrt(kws.length || 1)).toFixed(2);
+      if (score > 0) hits.push({ path: path.relative(this.localPath, f), score });
+    }
+    hits.sort((a, b) => b.score - a.score);
+    return hits.slice(0, k);
+  }
+
+  // ── Meta-mapping helpers (mirrors arb_knowledge.py iter_mappings) ───────────
+
+  /**
+   * listTags() — enumerate all .meta.json across the external repo.
+   * Returns same format as arb_knowledge.py list-tags.
+   *
+   * @returns {Array<{ directory, tag_id, tag_name, document_count, documents }>}
+   */
+  async listTags() {
+    if (!fsSync.existsSync(this.localPath)) await this.ensure();
+    const result = [];
+    const metas  = await walkFile(this.localPath, '.meta.json');
+    for (const mf of metas) {
+      let data = {};
+      try { data = JSON.parse(await fs.readFile(mf, 'utf8')); } catch { continue; }
+      const docs = data.documents || {};
+      result.push({
+        directory:      path.dirname(mf),
+        tag_id:         data.tag_id   || null,
+        tag_name:       data.tag_name || path.basename(path.dirname(mf)),
+        document_count: Object.keys(docs).length,
+        documents:      Object.keys(docs),
+      });
+    }
+    return result;
+  }
+
+  /**
+   * resolveId(knowledge_id) — reverse-lookup local file path from a knowledge_id.
+   * Mirrors arb_knowledge.py resolve sub-command.
+   *
+   * @param {string} knowledge_id
+   * @returns {{ local_path, tag_id, tag_name, filename } | null}
+   */
+  async resolveId(knowledge_id) {
+    if (!fsSync.existsSync(this.localPath)) return null;
+    const metas = await walkFile(this.localPath, '.meta.json');
+    for (const mf of metas) {
+      let data = {};
+      try { data = JSON.parse(await fs.readFile(mf, 'utf8')); } catch { continue; }
+      const docs = data.documents || {};
+      for (const [filename, kid] of Object.entries(docs)) {
+        if (kid === knowledge_id) {
+          return {
+            local_path: path.join(path.dirname(mf), filename),
+            tag_id:     data.tag_id   || null,
+            tag_name:   data.tag_name || null,
+            filename,
+          };
+        }
+      }
+    }
+    return null;
+  }
+
+  // ── MR/PR deposit flow ──────────────────────────────────────────────────────
+
+  /**
+   * High-level deposit + MR.
+   *
+   * Git flow mirrors arb_knowledge.py deposit_flow():
+   *   1. stash dirty changes (if any)
+   *   2. checkout targetBranch + pull
+   *   3. create branch deposit/<slug>-<date>
+   *   4. stash pop
+   *   5. copy files from workspace knowledge/ (or legacy devflow/knowledge/) into external repo
+   *   6. update .meta.json for new (A) files — ID left empty for CI to backfill
+   *   7. add + commit + push
+   *   8. create MR/PR via API
+   *   9. return { branch, url, status, commit_hash? }
+   *
+   * @param {string} projectRoot
+   * @param {{ items, slug, dryRun }} opts
+   */
+  async depositAndCreateMr(projectRoot, { items, slug, dryRun = false }) {
+    await this.ensure();
+
+    const date     = new Date().toISOString().slice(0, 10).replace(/-/g, '');
+    const safeslug = slug.replace(/[^a-zA-Z0-9-]/g, '-').replace(/-{2,}/g, '-').slice(0, 40).replace(/-$/, '');
+    const branch   = `deposit/${safeslug}-${date}`;
+    const target   = this.targetBranch;
+
+    if (dryRun) {
+      return {
+        branch, url: null, status: 'dry_run',
+        message: `would clone/pull ${this.repoUrl}, create branch '${branch}', copy ${(items || []).filter((p) => p.action !== 'noop').length} file(s), then open MR`,
+      };
+    }
+
+    // ── 1. Stash any dirty work-in-progress in the external clone ─────────────
+    const dirtyBefore = this._git(['status', '--porcelain']).stdout.trim();
+    const stashLabel  = `devflow-deposit-${branch}`;
+    if (dirtyBefore) {
+      this._git(['stash', 'push', '-u', '-m', stashLabel]);
+    }
+
+    // ── 2. Checkout targetBranch + pull latest ────────────────────────────────
+    this._git(['checkout', target]);
+    this._git(['pull', 'origin', target, '--ff-only']);
+
+    // ── 3. Create deposit branch (delete first if exists) ─────────────────────
+    const branchExistsR = this._git(['rev-parse', '--verify', branch]);
+    if (branchExistsR.status === 0) {
+      this._git(['branch', '-D', branch]);
+    }
+    this._git(['checkout', '-b', branch]);
+
+    // ── 4. Restore stash ──────────────────────────────────────────────────────
+    if (dirtyBefore) {
+      const popR = this._git(['stash', 'pop']);
+      if (popR.status !== 0) {
+        // Conflict during stash pop — leave on the branch for manual resolution.
+      }
+    }
+
+    // ── 5. Copy knowledge files from workspace knowledge/ → external repo ─────
+    const actionable  = (items || []).filter((p) => p.action !== 'noop');
+    for (const p of actionable) {
+      const srcKbDir = p.knowledgeRoot || inferKnowledgeRoot(projectRoot, p.targetAbs);
+      const rel  = path.relative(srcKbDir, p.targetAbs);
+      const dest = path.join(this.localPath, rel);
+      await fs.mkdir(path.dirname(dest), { recursive: true });
+      await fs.copyFile(p.targetAbs, dest);
+    }
+
+    // ── 6. Update .meta.json for new (A) files — knowledge_id left empty ──────
+    const newFiles = actionable.filter((p) => p.action === 'create');
+    if (newFiles.length) {
+      await this._updateMetaJsonForNew(projectRoot, newFiles);
+    }
+
+    // ── 7. Check for changes ──────────────────────────────────────────────────
+    const statusOut = this._git(['status', '--porcelain']).stdout.trim();
+    if (!statusOut) {
+      this._git(['checkout', target]);
+      return { branch: null, url: null, status: 'no_changes' };
+    }
+
+    // ── 8. Add + commit + push ────────────────────────────────────────────────
+    this._git(['add', '-A']);
+    const counts    = { A: 0, M: 0 };
+    for (const p of actionable) {
+      if (p.action === 'create') counts.A++;
+      else if (p.action === 'update') counts.M++;
+    }
+    const commitMsg = `chore(knowledge): [${slug}] deposit ${counts.A} added, ${counts.M} updated`;
+    const commitR   = this._git(['commit', '-m', commitMsg]);
+    if (commitR.status !== 0) {
+      this._git(['checkout', target]);
+      throw new Error(`git commit failed: ${(commitR.stderr || commitR.stdout || '').trim()}`);
+    }
+    const commitHash = this._git(['rev-parse', 'HEAD']).stdout.trim();
+
+    const pushR = this._git(['push', '-u', 'origin', branch]);
+    if (pushR.status !== 0) {
+      this._git(['checkout', target]);
+      throw new Error(`git push failed: ${(pushR.stderr || pushR.stdout || '').trim()}`);
+    }
+
+    // ── 9. Create MR/PR ───────────────────────────────────────────────────────
+    const { buildPrBody } = require('../../knowledge/mr.js');
+    const mrTitle = `[Knowledge] ${slug}`;
+    const mrBody  = buildPrBody(actionable, slug, branch, target);
+
+    let url = null;
+    try {
+      url = await this._createMrOrPr(branch, mrTitle, mrBody);
+    } catch (e) {
+      this._git(['checkout', target]);
+      return { branch, url: null, commit_hash: commitHash, status: 'pushed_no_mr', error: e.message };
+    }
+
+    this._git(['checkout', target]);
+    return { branch, url, commit_hash: commitHash, status: 'created' };
+  }
+
+  /**
+   * Deposit knowledge files directly to targetBranch — no feature branch, no MR.
+   * Use when team trusts CI-less fast path or the repo has no review gate.
+   */
+  async depositAndPush(projectRoot, { items, slug, dryRun = false }) {
+    await this.ensure();
+
+    const actionable = (items || []).filter((p) => p.action !== 'noop');
+    if (!actionable.length) return { status: 'no_changes' };
+
+    const target = this.targetBranch;
+
+    if (dryRun) {
+      return {
+        status: 'dry_run',
+        message: `would push ${actionable.length} file(s) directly to ${target} on ${this.repoUrl}`,
+      };
+    }
+
+    // 1. Ensure on targetBranch + latest
+    this._git(['checkout', target]);
+    this._git(['pull', 'origin', target, '--ff-only']);
+
+    // 2. Copy files
+    for (const p of actionable) {
+      const srcKbDir = p.knowledgeRoot || inferKnowledgeRoot(projectRoot, p.targetAbs);
+      const rel  = path.relative(srcKbDir, p.targetAbs);
+      const dest = path.join(this.localPath, rel);
+      await fs.mkdir(path.dirname(dest), { recursive: true });
+      await fs.copyFile(p.targetAbs, dest);
+    }
+
+    // 3. Update .meta.json for new files
+    const newFiles = actionable.filter((p) => p.action === 'create');
+    if (newFiles.length) await this._updateMetaJsonForNew(projectRoot, newFiles);
+
+    // 4. Check for changes
+    const statusOut = this._git(['status', '--porcelain']).stdout.trim();
+    if (!statusOut) return { status: 'no_changes' };
+
+    // 5. Commit + push directly to targetBranch
+    this._git(['add', '-A']);
+    const counts = { A: 0, M: 0 };
+    for (const p of actionable) {
+      if (p.action === 'create') counts.A++;
+      else if (p.action === 'update') counts.M++;
+    }
+    const commitMsg = `chore(knowledge): [${slug}] deposit ${counts.A} added, ${counts.M} updated`;
+    const commitR   = this._git(['commit', '-m', commitMsg]);
+    if (commitR.status !== 0) {
+      throw new Error(`git commit failed: ${(commitR.stderr || commitR.stdout || '').trim()}`);
+    }
+    const commitHash = this._git(['rev-parse', 'HEAD']).stdout.trim();
+
+    const pushR = this._git(['push', 'origin', target]);
+    if (pushR.status !== 0) {
+      throw new Error(`git push failed: ${(pushR.stderr || pushR.stdout || '').trim()}`);
+    }
+
+    return { status: 'pushed', commit_hash: commitHash, branch: target };
+  }
+
+  // ── .meta.json helpers ──────────────────────────────────────────────────────
+
+  /**
+   * For each newly created file, add an entry to the .meta.json in its category
+   * directory inside the external repo, with an empty knowledge_id (CI backfills).
+   * Mirrors arb_knowledge.py Step 5 behavior for A (add) operations.
+   */
+  async _updateMetaJsonForNew(projectRoot, newItems) {
+    for (const p of newItems) {
+      const srcKbDir  = p.knowledgeRoot || inferKnowledgeRoot(projectRoot, p.targetAbs);
+      const rel       = path.relative(srcKbDir, p.targetAbs);
+      const destAbs   = path.join(this.localPath, rel);
+      const catDir    = path.dirname(destAbs);
+      const metaFile  = path.join(catDir, '.meta.json');
+      const filename  = path.basename(destAbs);
+
+      let meta = {};
+      if (fsSync.existsSync(metaFile)) {
+        try { meta = JSON.parse(await fs.readFile(metaFile, 'utf8')); } catch { meta = {}; }
+      } else {
+        // Bootstrap a minimal .meta.json if the category dir is new.
+        meta = {
+          tag_id:   null,
+          tag_name: path.basename(catDir),
+          documents: {},
+        };
+      }
+
+      meta.documents = meta.documents || {};
+      if (!(filename in meta.documents)) {
+        meta.documents[filename] = '';  // empty = CI will backfill after WeKnora upload
+      }
+      await fs.writeFile(metaFile, JSON.stringify(meta, null, 2) + '\n', 'utf8');
+    }
+  }
+
+  // ── Git helpers ─────────────────────────────────────────────────────────────
+
+  /**
+   * Clone if absent, else checkout targetBranch + pull latest.
+   * Mirrors arb_knowledge.py ensure_clone().
+   */
+  async ensure() {
+    if (!this.repoUrl)   throw new Error('kb-git: repoUrl not configured');
+    if (!this.localPath) throw new Error('kb-git: localPath not configured');
+
+    if (!fsSync.existsSync(path.join(this.localPath, '.git'))) {
+      await fs.mkdir(path.dirname(this.localPath), { recursive: true });
+      const r = cp.spawnSync('git', ['clone', this.repoUrl, this.localPath], { encoding: 'utf8' });
+      if (r.status !== 0) throw new Error(`git clone failed: ${(r.stderr || r.stdout || '').trim()}`);
+    } else {
+      this._git(['fetch', 'origin', '--no-tags']);
+      const coR = this._git(['checkout', this.targetBranch]);
+      if (coR.status !== 0) {
+        // Already on target branch or detached HEAD — ignore.
+      }
+      this._git(['pull', 'origin', this.targetBranch, '--ff-only']);
+    }
+  }
+
+  _git(args) {
+    return cp.spawnSync('git', args, {
+      cwd: this.localPath, encoding: 'utf8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+  }
+
+  _gitCommitPush(msg) {
+    this._git(['add', '-A']);
+    this._git(['commit', '-m', msg]);
+    this._git(['push', 'origin', this.targetBranch]);
+  }
+
+  // ── MR / PR API ─────────────────────────────────────────────────────────────
+
+  async _createMrOrPr(branch, title, body) {
+    const isGitHub = this.githubToken || this.githubRepo ||
+      /github\.com/i.test(this.repoUrl);
+
+    if (isGitHub) return this._createGithubPr(branch, title, body);
+    if (this.gitlabApiUrl && this.gitlabToken && this.projectId) {
+      return this._createGitlabMr(branch, title, body);
+    }
+    return this._createViaGlabCli(branch, title, body);
+  }
+
+  /** Mirrors arb_knowledge.py create_mr() using the GitLab REST API. */
+  async _createGitlabMr(branch, title, description) {
+    const projectPath = encodeURIComponent(this.projectId);
+    const url         = `${this.gitlabApiUrl}/projects/${projectPath}/merge_requests`;
+    const body        = {
+      source_branch:        branch,
+      target_branch:        this.targetBranch,
+      title,
+      description,
+      remove_source_branch: true,
+    };
+    if (this.mrReviewers.length) {
+      body.description += `\n\n/assign_reviewer ${this.mrReviewers.join(' @')}`;
+    }
+    const resp = await this._httpPost(url, body, { 'PRIVATE-TOKEN': this.gitlabToken });
+    return resp.web_url || null;
+  }
+
+  async _createGithubPr(branch, title, body) {
+    let repo = this.githubRepo;
+    if (!repo) {
+      const m = this.repoUrl.match(/github\.com[/:]([^/]+\/[^/]+?)(?:\.git)?$/i);
+      if (!m) throw new Error('kb-git: cannot determine GitHub owner/repo from repoUrl');
+      repo = m[1];
+    }
+    const url     = `https://api.github.com/repos/${repo}/pulls`;
+    const payload = { title, body, head: branch, base: this.targetBranch };
+    const resp    = await this._httpPost(url, payload, {
+      Authorization:          `Bearer ${this.githubToken}`,
+      'User-Agent':           'devflow-kit',
+      'X-GitHub-Api-Version': '2022-11-28',
+    });
+    return resp.html_url || null;
+  }
+
+  async _createViaGlabCli(branch, title, body) {
+    const r = cp.spawnSync('glab', [
+      'mr', 'create',
+      '--title',         title,
+      '--description',   body,
+      '--source-branch', branch,
+      '--target-branch', this.targetBranch,
+      '--yes',
+    ], { cwd: this.localPath, encoding: 'utf8' });
+    if (r.status !== 0) throw new Error(`glab mr create failed: ${(r.stderr || r.stdout || '').trim()}`);
+    return (r.stdout || '').trim().split('\n').filter(Boolean).pop() || null;
+  }
+
+  _httpPost(url, body, headers = {}) {
+    return new Promise((resolve, reject) => {
+      const parsed = new URL(url);
+      const data   = JSON.stringify(body);
+      const opts   = {
+        hostname: parsed.hostname,
+        port:     parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
+        path:     parsed.pathname + (parsed.search || ''),
+        method:   'POST',
+        headers:  {
+          'Content-Type':   'application/json',
+          'Content-Length': Buffer.byteLength(data),
+          ...headers,
+        },
+      };
+      const lib = parsed.protocol === 'https:' ? https : http;
+      const req = lib.request(opts, (res) => {
+        let raw = '';
+        res.on('data', (c) => { raw += c; });
+        res.on('end', () => {
+          if (res.statusCode >= 400) {
+            reject(new Error(`HTTP ${res.statusCode}: ${raw.slice(0, 300)}`));
+            return;
+          }
+          try { resolve(JSON.parse(raw)); }
+          catch { resolve({}); }
+        });
+      });
+      req.on('error', reject);
+      req.setTimeout(15000, () => { req.destroy(new Error('timeout')); });
+      req.write(data);
+      req.end();
+    });
+  }
+}
+
+// ── Utilities ───────────────────────────────────────────────────────────────
+
+function expandHome(p) {
+  if (!p) return p;
+  if (p.startsWith('~/') || p === '~') return path.join(os.homedir(), p.slice(1));
+  return p;
+}
+
+function inferKnowledgeRoot(projectRoot, filePath) {
+  const legacy = path.join(projectRoot, 'devflow', 'knowledge');
+  if (filePath.startsWith(legacy + path.sep)) return legacy;
+  const marker = `${path.sep}knowledge${path.sep}`;
+  const idx = filePath.lastIndexOf(marker);
+  if (idx >= 0) return filePath.slice(0, idx + marker.length - 1);
+  return legacy;
+}
+
+async function walkMd(dir) {
+  return walkFile(dir, '.md');
+}
+
+async function walkFile(dir, suffix) {
+  const out = [];
+  if (!fsSync.existsSync(dir)) return out;
+  const stack = [dir];
+  while (stack.length) {
+    const cur  = stack.pop();
+    const ents = await fs.readdir(cur, { withFileTypes: true });
+    for (const e of ents) {
+      if (e.name === '.git') continue;
+      const p = path.join(cur, e.name);
+      if (e.isDirectory()) stack.push(p);
+      else if (e.isFile() && e.name.endsWith(suffix)) out.push(p);
+    }
+  }
+  return out;
+}
+
+module.exports = { default: KbGitProvider };