kld-sdd 2.6.16 → 2.6.21

package/skywalk-sdd/kb-upload.cjs

@@ -0,0 +1,505 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * kb-upload.cjs — 知识库上传脚本
+ *
+ * 支持两种上传模式：
+ *   1. 文件夹上传：自动跑语义管线 → 生成元数据 → 打包 zip → 上传
+ *   2. Zip 上传：直接上传已有 zip 包
+ *
+ * 用法：
+ *   node skywalk-sdd/kb-upload.cjs --folder=<path> [--minimum-level=propose-spec] [--validate-only]
+ *   node skywalk-sdd/kb-upload.cjs --package=<zip-path>
+ *   node skywalk-sdd/kb-upload.cjs --folder=<path> --project=<spec-root> --change=<name>
+ *
+ * project_id 解析：
+ *   1. $SPEC_ROOT/skywalk-sdd/project-identity.json 的 project_id
+ *   2. 都没有 → 报错：请运行 /opsx-kb-config 配置知识库
+ */
+
+const fs = require('fs');
+const path = require('path');
+const {
+  parseArgs,
+  readJson,
+  readJsonSafe,
+  writeJsonAtomic,
+  ensureDir,
+  copyDirRecursive,
+  resolveSpecRootFast,
+  loadKbState,
+  uploadFile,
+  httpGetJson,
+} = require('./lib/shared.cjs');
+
+/**
+ * 读取 kb-state.json 中指定 change 的 archive 缓存。
+ * @param {string} specRoot
+ * @param {string} changeName
+ * @returns {object|null} { archive_id, content_hash, uploaded_at }
+ */
+function readArchiveCache(specRoot, changeName) {
+  const directPath = path.join(specRoot, 'kb-state.json');
+  const legacyPath = path.join(specRoot, '..', '.shared', 'kb-state.json');
+  const statePath = fs.existsSync(directPath) ? directPath : legacyPath;
+  const state = readJsonSafe(statePath);
+  if (!state || !state.archiveCache) return null;
+  return state.archiveCache[changeName] || null;
+}
+
+/**
+ * 更新 kb-state.json 中的 archiveCache（原子写入）。
+ * @param {string} specRoot
+ * @param {string} changeName
+ * @param {string} archiveId
+ * @param {string} contentHash
+ */
+function updateKbStateCache(specRoot, changeName, archiveId, contentHash) {
+  const directPath = path.join(specRoot, 'kb-state.json');
+  const legacyPath = path.join(specRoot, '..', '.shared', 'kb-state.json');
+  const statePath = fs.existsSync(directPath) ? directPath : legacyPath;
+  const state = readJsonSafe(statePath) || {};
+  if (!state.archiveCache) state.archiveCache = {};
+  state.archiveCache[changeName] = {
+    archive_id: archiveId,
+    content_hash: contentHash,
+    uploaded_at: new Date().toISOString(),
+  };
+  writeJsonAtomic(statePath, state);
+  console.log(`✓ 已更新本地缓存: ${changeName} → ${archiveId.slice(0, 24)}...`);
+}
+
+// ── 常量 ──────────────────────────────────────────────
+
+const MINIMUM_LEVELS = {
+  'propose-spec': ['proposal.md', 'spec.md'],
+  'design': ['proposal.md', 'spec.md', 'design.md'],
+  'task': ['proposal.md', 'spec.md', 'design.md', 'tasks.md'],
+  'full': ['proposal.md', 'spec.md', 'design.md', 'tasks.md'],
+};
+
+const DEFAULT_LEVEL = 'propose-spec';
+
+// ── 工具函数 ──────────────────────────────────────────
+
+function resolveSpecRoot(explicit) {
+  if (explicit) return path.resolve(explicit);
+  return resolveSpecRootFast(__dirname);
+}
+
+/**
+ * 解析 project_id
+ * 优先级：project-identity.json → 报错
+ */
+function resolveProjectId(specRoot) {
+  const identityPath = path.join(specRoot, 'skywalk-sdd', 'project-identity.json');
+  if (fs.existsSync(identityPath)) {
+    const identity = readJson(identityPath);
+    if (identity.project_id) {
+      return identity.project_id;
+    }
+  }
+  throw new Error(
+    '无法确定 project_id。project-identity.json 不存在或无 project_id 字段。\n' +
+    '请运行 /opsx-kb-config 配置知识库。'
+  );
+}
+
+/**
+ * 读取 KB 配置（api, apiKey, tenantKey, targets）
+ * 使用 shared.cjs 的 loadKbState，支持 BOM 剥离和多路径搜索
+ */
+function readKbState(specRoot) {
+  // 优先尝试 specRoot 根目录下的 kb-state.json（新路径）
+  const directPath = path.join(specRoot, 'kb-state.json');
+  // 兼容旧路径：specRoot 同级 .shared/kb-state.json
+  const legacyPath = path.join(specRoot, '..', '.shared', 'kb-state.json');
+  if (fs.existsSync(directPath)) {
+    try {
+      return readJson(directPath);
+    } catch (e) {
+      throw new Error(
+        `KB 配置解析失败 (${directPath}): ${e.message}\n` +
+        '请运行 /opsx-kb-config 重新配置知识库。'
+      );
+    }
+  }
+  // 退回到多路径搜索（从 specRoot 向上查找）
+  const { config, path: foundPath, warnings } = loadKbState(specRoot);
+  if (warnings.length > 0) {
+    for (const w of warnings) console.warn(`⚠️  ${w}`);
+  }
+  if (!config) {
+    throw new Error(
+      'KB 配置不存在（kb-state.json 缺失）。\n' +
+      '请运行 /opsx-kb-config 配置知识库。'
+    );
+  }
+  return config;
+}
+
+/**
+ * 校验文件夹必需文件
+ */
+function validateRequiredFiles(folderPath, minimumLevel) {
+  const required = MINIMUM_LEVELS[minimumLevel] || MINIMUM_LEVELS[DEFAULT_LEVEL];
+  const missing = required.filter(f => !fs.existsSync(path.join(folderPath, f)));
+  if (missing.length > 0) {
+    throw new Error(
+      `[${minimumLevel}] 缺少必需文件: ${missing.join(', ')}\n` +
+      `目录: ${folderPath}\n` +
+      `等级说明:\n` +
+      `  propose-spec: proposal.md + spec.md\n` +
+      `  design:       + design.md\n` +
+      `  task:         + tasks.md\n` +
+      `  full:         全部 4 个`
+    );
+  }
+  console.log(`✓ 文件校验通过 [${minimumLevel}]: ${required.join(', ')}`);
+}
+
+/**
+ * 本地预检 project_id === spaceKey
+ */
+function preCheckProjectId(projectId, state) {
+  if (!state.targets || !state.targets[0]) {
+    throw new Error('KB 配置中无 targets，请运行 /opsx-kb-config 配置知识库。');
+  }
+  const spaceKey = state.targets[0].spaceKey;
+  if (!spaceKey) {
+    console.warn('⚠️  kb-state.json targets[0].spaceKey 为空，跳过本地预检');
+    return;
+  }
+  if (projectId !== spaceKey) {
+    throw new Error(
+      `PROJECT_SPACE_MISMATCH 预检失败：\n` +
+      `  project_id (project-identity.json): ${projectId}\n` +
+      `  spaceKey    (kb-state.json):        ${spaceKey}\n` +
+      `请运行 /opsx-kb-config 更新配置。`
+    );
+  }
+  console.log(`✓ project_id 预检通过: ${projectId} === ${spaceKey}`);
+}
+
+/**
+ * 文件夹模式：构建 Archive Package（生成元数据 + zip）
+ *
+ * 流程：
+ *   1. 运行 semantic-reconcile（生成 working-ontology.json）
+ *   2. 创建临时 staging 目录
+ *   3. 复制源文件到 staging
+ *   4. createArchiveSnapshot（生成 archive-ontology.json）
+ *   5. materializeArchivePackage（生成 manifest + canonical-facts + conversion-report + zip）
+ *   6. 返回 zip 路径
+ */
+function buildPackageFromFolder(specRoot, changeName, options) {
+  const runtime = require('./ontology/runtime.cjs');
+  const archivePackage = require('./ontology/archive-package.cjs');
+
+  // 1. 语义管线
+  console.log('→ 运行语义管线 (semantic-reconcile)...');
+  const semanticResult = runtime.reconcileChange(specRoot, changeName, {
+    profile: 'auto',
+    markPending: true,
+  });
+  if (!semanticResult.valid) {
+    const errors = semanticResult.diagnostics
+      .filter(d => d.severity === 'error')
+      .map(d => `${d.code}: ${d.message}`);
+    throw new Error(`语义校验失败:\n${errors.join('\n')}`);
+  }
+  console.log(`✓ 语义校验通过 (profile: ${semanticResult.profile}, revision: ${semanticResult.revision})`);
+
+  // 2. 创建临时 staging 目录
+  const stagingDir = path.join(
+    fs.mkdtempSync(path.join(require('os').tmpdir(), 'kb-upload-')),
+  );
+  const zipPath = path.join(stagingDir, `${changeName}.zip`);
+
+  try {
+    // 3. 复制源文件到 staging
+    const changeDir = path.join(specRoot, 'openspec', 'changes', changeName);
+    copyDirRecursive(changeDir, stagingDir);
+
+    // 4. 创建 archive snapshot（生成 archive-ontology.json）
+    runtime.createArchiveSnapshot(specRoot, changeName, stagingDir, semanticResult.state);
+    console.log('✓ 已生成 archive-ontology.json');
+
+    // 5. 生成 manifest + canonical-facts + conversion-report + zip
+    const projectIdentity = archivePackage.ensureProjectIdentity(specRoot);
+    const result = archivePackage.materializeArchivePackage({
+      projectRoot: specRoot,
+      archiveDir: stagingDir,
+      archivePath: stagingDir,
+      changeName: changeName,
+      packagePath: zipPath,
+      reason: options.reason || '随时入库（文件夹上传）',
+      method: 'kb-upload-folder',
+    });
+
+    console.log(`✓ Archive Package 已生成`);
+    console.log(`  archive_id: ${result.manifest.archive_id}`);
+    console.log(`  project_id: ${result.manifest.project_id}`);
+    console.log(`  files: ${result.manifest.files.length}`);
+
+    return { zipPath, manifest: result.manifest, stagingDir };
+  } catch (err) {
+    // 出错时清理 staging
+    fs.rmSync(stagingDir, { recursive: true, force: true });
+    throw err;
+  }
+}
+
+/**
+ * 上传 zip 到 KB（使用 Node.js HTTP，替代 curl）
+ */
+async function uploadZip(zipPath, state, options) {
+  const target = options.spaceId && options.kbId
+    ? { spaceId: options.spaceId, kbId: options.kbId }
+    : state.targets[0];
+
+  if (!target || !target.spaceId || !target.kbId) {
+    throw new Error('未指定目标 Space/KB，请运行 /opsx-kb-config 配置知识库。');
+  }
+
+  const apiUrl = state.api || 'http://localhost:8090/api';
+  const uploadUrl = `${apiUrl}/v1/spaces/${target.spaceId}/knowledge-bases/${target.kbId}/ingestions`;
+
+  console.log(`→ 上传到 KB: ${target.spaceName || target.spaceId}/${target.kbName || target.kbId}`);
+  console.log(`  URL: ${uploadUrl}`);
+  console.log(`  ZIP: ${zipPath} (${(fs.statSync(zipPath).size / 1024).toFixed(1)} KB)`);
+
+  try {
+    const result = await uploadFile(uploadUrl, zipPath, state.apiKey);
+    return result;
+  } catch (err) {
+    throw new Error(`上传失败: ${err.message}`);
+  }
+}
+
+// ── 主流程 ────────────────────────────────────────────
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+
+  if (!args.folder && !args.package) {
+    console.error('用法:');
+    console.error('  node skywalk-sdd/kb-upload.cjs --folder=<path> [--minimum-level=propose-spec] [--validate-only]');
+    console.error('  node skywalk-sdd/kb-upload.cjs --package=<zip-path>');
+    console.error('  node skywalk-sdd/kb-upload.cjs --folder=<path> --project=<spec-root> --change=<name>');
+    process.exit(1);
+  }
+
+  const minimumLevel = args['minimum-level'] || DEFAULT_LEVEL;
+  if (!MINIMUM_LEVELS[minimumLevel]) {
+    console.error(`无效的 --minimum-level: ${minimumLevel}`);
+    console.error(`可选值: ${Object.keys(MINIMUM_LEVELS).join(', ')}`);
+    process.exit(1);
+  }
+
+  try {
+    // ── Zip 模式 ──
+    if (args.package) {
+      const zipPath = path.resolve(args.package);
+      if (!fs.existsSync(zipPath)) {
+        throw new Error(`zip 文件不存在: ${zipPath}`);
+      }
+      console.log(`模式: Zip 上传`);
+      console.log(`  ZIP: ${zipPath}`);
+
+      const specRoot = resolveSpecRoot(args.project);
+      const state = readKbState(specRoot);
+
+      if (!state.apiKey) {
+        throw new Error('KB 配置中无 apiKey，请运行 /opsx-kb-config 配置知识库。');
+      }
+
+      const result = await uploadZip(zipPath, state, {
+        spaceId: args['space-id'],
+        kbId: args['kb-id'],
+      });
+
+      console.log('\n── 上传结果 ──');
+      console.log(JSON.stringify(result, null, 2));
+      return;
+    }
+
+    // ── 文件夹模式 ──
+    const specRoot = resolveSpecRoot(args.project);
+
+    // 解析 changeName
+    let changeName;
+    let folderPath;
+    if (args.change) {
+      changeName = args.change;
+      folderPath = path.join(specRoot, 'openspec', 'changes', changeName);
+    } else {
+      folderPath = path.resolve(args.folder);
+      changeName = path.basename(folderPath);
+    }
+
+    if (!fs.existsSync(folderPath)) {
+      throw new Error(`变更目录不存在: ${folderPath}`);
+    }
+
+    console.log(`模式: 文件夹上传`);
+    console.log(`  目录: ${folderPath}`);
+    console.log(`  变更: ${changeName}`);
+    console.log(`  等级: ${minimumLevel}`);
+    console.log(`  Spec Root: ${specRoot}`);
+    console.log('');
+
+    // 1. 校验必需文件
+    validateRequiredFiles(folderPath, minimumLevel);
+
+    // 2. 解析 project_id
+    const projectId = resolveProjectId(specRoot);
+    console.log(`✓ project_id: ${projectId}`);
+
+    // 3. 读取 KB 配置 + 预检
+    const state = readKbState(specRoot);
+    if (!state.apiKey) {
+      throw new Error('KB 配置中无 apiKey，请运行 /opsx-kb-config 配置知识库。');
+    }
+    preCheckProjectId(projectId, state);
+
+    // 4. 仅校验模式（含语义校验）
+    if (args['validate-only']) {
+      const runtime = require('./ontology/runtime.cjs');
+      console.log('→ 运行语义管线 (semantic-reconcile)...');
+      const semanticResult = runtime.reconcileChange(specRoot, changeName, {
+        profile: 'auto',
+        markPending: true,
+      });
+      if (!semanticResult.valid) {
+        const errors = semanticResult.diagnostics
+          .filter(d => d.severity === 'error')
+          .map(d => `${d.code}: ${d.message}`);
+        throw new Error(`语义校验失败:\n${errors.join('\n')}`);
+      }
+      console.log(`✓ 语义校验通过 (profile: ${semanticResult.profile}, revision: ${semanticResult.revision})`);
+      console.log('\n✓ 校验完成（--validate-only，未上传）');
+      return;
+    }
+
+    // 5. 构建 Archive Package
+    console.log('');
+    const { zipPath, manifest, stagingDir } = buildPackageFromFolder(specRoot, changeName, {
+      reason: args.reason,
+    });
+
+    try {
+      // === Step 5.1: 本地缓存快速检查 ===
+      const cache = readArchiveCache(specRoot, changeName);
+      if (cache && cache.archive_id === manifest.archive_id) {
+        console.log('\n✓ 内容无变更（archive_id 与上次成功上传一致），跳过上传');
+        console.log(`  cached archive_id: ${cache.archive_id}`);
+        console.log(`  uploaded_at: ${cache.uploaded_at}`);
+        console.log('\n✓ 完成（本地缓存命中）');
+        return;
+      }
+
+      // 解析上传目标
+      const target = (args['space-id'] && args['kb-id'])
+        ? { spaceId: args['space-id'], kbId: args['kb-id'] }
+        : state.targets[0];
+
+      if (!target || !target.spaceId || !target.kbId) {
+        throw new Error('未指定目标 Space/KB，请运行 /opsx-kb-config 配置知识库。');
+      }
+
+      const apiUrl = state.api || 'http://localhost:8090/api';
+
+      // === Step 5.5: KB 预检（网络级幂等确认）===
+      const archiveId = manifest.archive_id;
+      const contentHash = manifest.content_hash;
+      console.log(`→ Pre-check: archive_id=${archiveId}`);
+      console.log(`  content_hash=${contentHash}`);
+
+      const checkUrl = `${apiUrl}/v1/spaces/${target.spaceId}/knowledge-bases/${target.kbId}/ingestions/check`
+        + `?archiveId=${encodeURIComponent(archiveId)}`
+        + `&contentHash=${encodeURIComponent(contentHash)}`;
+
+      try {
+        const checkResult = await httpGetJson(checkUrl, state.apiKey);
+
+        if (checkResult && checkResult.data && checkResult.data.idempotent) {
+          console.log('\n✓ KB 中已存在相同 archive（幂等命中），跳过上传');
+          console.log(`  snapshot_id: ${checkResult.data.snapshotId}`);
+          updateKbStateCache(specRoot, changeName, archiveId, contentHash);
+          console.log('\n✓ 完成（KB 预检命中）');
+          return;
+        }
+
+        if (checkResult && checkResult.data && checkResult.data.exists && !checkResult.data.idempotent) {
+          throw new Error(
+            `ARCHIVE_IMMUTABILITY_VIOLATION: archive_id 已存在但 content_hash 不同\n` +
+            `  archive_id: ${archiveId}\n` +
+            `  如果内容确实变更，请确认实体身份块已正确更新`
+          );
+        }
+
+        console.log('✓ Pre-check 通过（KB 中无此 archive，继续上传）');
+      } catch (checkErr) {
+        // 降级：KB 后端未部署 /check 接口（404）→ 继续上传
+        if (checkErr.message && checkErr.message.includes('HTTP 404')) {
+          console.log('⚠️  KB 后端未部署 /check 接口，降级为直接上传');
+        } else {
+          throw checkErr;
+        }
+      }
+
+      // 6. 上传
+      console.log('');
+      const result = await uploadZip(zipPath, state, {
+        spaceId: target.spaceId,
+        kbId: target.kbId,
+      });
+
+      console.log('\n── 上传结果 ──');
+      console.log(JSON.stringify(result, null, 2));
+
+      // 7. 输出 manifest 摘要
+      console.log('\n── Archive Package 摘要 ──');
+      console.log(`  archive_id: ${manifest.archive_id}`);
+      console.log(`  project_id: ${manifest.project_id}`);
+      console.log(`  change_id:  ${manifest.change_id}`);
+      console.log(`  files:      ${manifest.files.length}`);
+
+      // === 上传成功后更新本地缓存 ===
+      updateKbStateCache(specRoot, changeName, archiveId, contentHash);
+
+      // === 投影状态提示 ===
+      // 上传成功后，KB 后端异步执行 vector embedding + AGE graph 投影。
+      // 正常 SDD 流程中，Spec→Design 等阶段间有用户手动触发的时间间隔，投影通常在此期间完成。
+      const projStatus = result?.data?.projectionStatus || result?.projectionStatus || '';
+      const jobStatus = result?.data?.status || result?.status || '';
+
+      if (projStatus === 'complete') {
+        console.log('\n✓ 投影已完成，向量/图检索可用');
+      } else if (projStatus === 'degraded') {
+        console.log('\n⚠️  投影完成但有降级（部分通道不可用，FTS 仍可用）');
+      } else if (projStatus === 'disabled') {
+        console.log('\nℹ️  投影已禁用（FTS-only 模式）');
+      } else if (projStatus && projStatus !== 'complete') {
+        console.log(`\nℹ️  投影进行中 (${projStatus})，向量/图检索暂不可用`);
+        console.log('  → 正常 SDD 流程中阶段间间隔通常足够投影完成');
+        console.log('  → 如后续查询 degraded，可用 `context-client.cjs --check-only` 确认 KB 可用性');
+      } else if (jobStatus === 'succeeded') {
+        console.log('\n✓ 入库成功');
+      }
+
+    } finally {
+      // 清理临时目录
+      fs.rmSync(stagingDir, { recursive: true, force: true });
+      console.log('\n✓ 临时文件已清理');
+    }
+
+  } catch (err) {
+    console.error(`\n❌ ${err.message}`);
+    process.exit(1);
+  }
+}
+
+main();