@dsh-plugin/dsh-synapse 0.0.0-stage → 0.1.0-dev.41044818627

package/src/index.ts

@@ -0,0 +1,227 @@
+import { mkdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { template, validateConfig } from "./config.js";
+import { pathsFor, SkillStore, safePath } from "./files.js";
+import { Inference } from "./runtime.js";
+import { Memory, parseHermes } from "./memory.js";
+import { McpManager } from "./mcp.js";
+import { Agent } from "./agent.js";
+import { Gateway } from "./gateway.js";
+import { Bot } from "./bot.js";
+import { Reviews } from "./github.js";
+import { json, body, browserAuthorized } from "./http.js";
+import type { SynapseContext, SynapseConfig, ErrorSink } from "./types.js";
+export const name = "@dsh-plugin/dsh-synapse";
+export const inject = ["dshLoader"];
+export { template };
+export async function createService(
+  ctx: SynapseContext,
+  c: SynapseConfig,
+  onError: ErrorSink,
+) {
+  validateConfig(c);
+  const paths = pathsFor(ctx, c);
+  await safePath(paths.profile, "data", "dsh-synapse");
+  await mkdir(paths.data, { recursive: true });
+  const config = () => c,
+    inference = new Inference(paths.data),
+    skills = new SkillStore(paths);
+  const memory = new Memory(
+      path.join(paths.data, "memory.sqlite"),
+      inference,
+      config,
+    ),
+    mcp = new McpManager(config);
+  const agent = new Agent({ config, paths, memory, skills, mcp, onError }),
+    gateway = new Gateway(agent, config, onError);
+  const bot = new Bot(paths, config, gateway, onError),
+    reviews = new Reviews(memory.db, agent, config, bot, onError);
+  const close = async () => {
+    await Promise.allSettled([
+      reviews.close(),
+      bot.stop(),
+      gateway.close(),
+      mcp.close(),
+      inference.close(),
+    ]);
+    memory.close();
+  };
+  try {
+    await gateway.start();
+    await bot.start();
+    void reviews.drain();
+    if (c.enabled && c.ragEnabled)
+      void inference.embed("初始化记忆模型").catch(onError);
+  } catch (e) {
+    await close();
+    throw e;
+  }
+  return {
+    paths,
+    config,
+    inference,
+    skills,
+    memory,
+    mcp,
+    agent,
+    gateway,
+    bot,
+    reviews,
+    close,
+  };
+}
+export async function apply(ctx: SynapseContext) {
+  const handle = await ctx.dshLoader.config.register<SynapseConfig>(
+    ctx,
+    template,
+  );
+  let service,
+    disposed = false,
+    lastError = "",
+    queue = Promise.resolve();
+  const onError = (e) => {
+    lastError = e.message;
+    ctx.logger?.warn?.(`[synapse] ${e.message}`);
+  };
+  const enqueue = (fn) => {
+    const result = queue.then(fn);
+    queue = result.catch(onError);
+    return result;
+  };
+  const reconfigure = (value) =>
+    enqueue(async () => {
+      if (disposed) return;
+      validateConfig(value);
+      pathsFor(ctx, value);
+      if (service) {
+        const old = service;
+        service = null;
+        await old.close();
+      }
+      service = await createService(ctx, value, onError);
+      lastError = "";
+    });
+  const off = handle.watch(
+    ({ value }) => void reconfigure(value).catch(() => {}),
+  );
+  const manage = async (req, res) => {
+    try {
+      if (!browserAuthorized(ctx, req, res)) return;
+      const url = new URL(req.url, "http://localhost");
+      const action = url.pathname.slice("/api/dsh-synapse/".length);
+      if (action === "status")
+        return json(res, {
+          running: Boolean(service),
+          bot: service?.bot.state || "stopped",
+          lastError,
+        });
+      if (!service)
+        return json(res, { error: lastError || "服务尚未就绪" }, 503);
+      // Serialize management with configuration changes; no writes into a closing DB.
+      await enqueue(async () => {
+        if (!service) throw new Error("服务正在重载");
+        if (req.method === "GET" && action === "skills")
+          return json(res, await service.skills.list());
+        if (req.method === "GET" && action === "reviews")
+          return json(res, service.reviews.list());
+        if (req.method === "GET" && action === "memory")
+          return json(
+            res,
+            service.memory.list(url.searchParams.get("channel") || "chat"),
+          );
+        if (req.method === "GET" && action === "features") {
+          const live = await service.bot.liveFeatures();
+          return json(
+            res,
+            service.config().features.map((f) => ({
+              id: f.id,
+              value: {
+                ...JSON.parse(f.json),
+                ...(live.find((x) => x.id === f.id)?.value || {}),
+              },
+            })),
+          );
+        }
+        if (req.method === "POST") {
+          const data = JSON.parse((await body(req)).toString() || "{}");
+          if (action === "skills/save")
+            return json(
+              res,
+              await service.skills.save(data.name, data.content, data.revision),
+            );
+          if (action === "skills/delete") {
+            await service.skills.remove(data.name, data.revision);
+            return json(res, { ok: true });
+          }
+          if (action === "mcp/tools")
+            return json(res, await service.mcp.list(data.name));
+          if (action === "memory/import") {
+            if (!service.config().ragEnabled) throw new Error("请先启用 RAG");
+            const items = parseHermes(data.markdown, data.channel);
+            return json(
+              res,
+              data.preview ? { items } : await service.memory.addMany(items),
+            );
+          }
+          if (action === "memory/search")
+            return json(
+              res,
+              await service.memory.search(String(data.query), data.channel),
+            );
+          if (action === "reviews/retry") {
+            service.reviews.retry(data.id);
+            return json(res, { ok: true });
+          }
+        }
+        json(res, { error: "Not found" }, 404);
+      });
+    } catch (e) {
+      if (!res.headersSent) json(res, { error: e.message }, e.status || 400);
+    }
+  };
+  const routes = [];
+  ctx.effect(() => () => {
+    disposed = true;
+    off();
+    for (const release of routes) release();
+    void enqueue(async () => {
+      await service?.close();
+      service = null;
+    });
+  });
+  routes.push(ctx.dshLoader.web.register("/api/dsh-synapse/", manage));
+  routes.push(
+    ctx.dshLoader.web.post("/hooks/dsh-synapse/github", (req, res) => {
+      void enqueue(async () => {
+        if (!service) return json(res, { error: "服务尚未就绪" }, 503);
+        await service.reviews.webhook(req, res);
+      }).catch((e) => {
+        if (!res.headersSent) json(res, { error: e.message }, e.status || 400);
+      });
+    }),
+  );
+  // A same-origin standalone panel is also available for older DSH clients.
+  routes.push(
+    ctx.dshLoader.web.get("/synapse", async (req, res) => {
+      if (!browserAuthorized(ctx, req, res)) return;
+      const html = await readFile(
+        new URL("./panel.html", import.meta.url),
+        "utf8",
+      );
+      res.writeHead(200, {
+        "Content-Type": "text/html; charset=utf-8",
+        "Content-Security-Policy":
+          "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; frame-ancestors 'self'",
+      });
+      res.end(html);
+    }),
+  );
+  routes.push(
+    ctx.dshLoader.web.get("/synapse-panel.js", async (req, res) => {
+      if (!browserAuthorized(ctx, req, res)) return;
+      res.writeHead(200, { "Content-Type": "text/javascript; charset=utf-8" });
+      res.end(await readFile(new URL("./panel.js", import.meta.url), "utf8"));
+    }),
+  );
+  await reconfigure(handle.get().value).catch(() => {});
+}

package/src/inference-worker.ts

@@ -0,0 +1,89 @@
+import { parentPort, workerData } from "node:worker_threads";
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+const require = createRequire(workerData.manifest);
+const runtime = await import(
+  pathToFileURL(require.resolve("@huggingface/transformers")).href
+);
+const { pipeline, env, AutoTokenizer, AutoModelForSequenceClassification } =
+  runtime.default ?? runtime;
+env.cacheDir = workerData.cache;
+env.allowLocalModels = false;
+let embedder, tokenizer, ranker;
+async function load(factory) {
+  try {
+    env.remoteHost = "https://huggingface.co/";
+    return await factory();
+  } catch (direct) {
+    env.remoteHost = "https://hf-mirror.com/";
+    try {
+      return await factory();
+    } catch (mirror) {
+      throw new Error(
+        `模型直连和镜像均失败：${direct.message}; ${mirror.message}`,
+      );
+    }
+  }
+}
+let queue = Promise.resolve();
+parentPort.on("message", (message) => {
+  queue = queue
+    .then(async () => {
+      const { id, method, args } = message;
+      try {
+        let value;
+        if (method === "embed") {
+          embedder ??= await load(() =>
+            pipeline("feature-extraction", "Xenova/bge-small-zh-v1.5", {
+              dtype: "q8",
+              device: "cpu",
+            }),
+          );
+          const output = await embedder(
+            (args.query ? "为这个句子生成表示以用于检索相关文章：" : "") +
+              args.text,
+            {
+              pooling: "cls",
+              normalize: true,
+              truncation: true,
+              max_length: 512,
+            },
+          );
+          value = Array.from(output.data);
+        } else if (method === "rerank") {
+          tokenizer ??= await load(() =>
+            AutoTokenizer.from_pretrained("Xenova/bge-reranker-base"),
+          );
+          ranker ??= await load(() =>
+            AutoModelForSequenceClassification.from_pretrained(
+              "Xenova/bge-reranker-base",
+              { dtype: "q8", device: "cpu" },
+            ),
+          );
+          value = [];
+          for (let i = 0; i < args.documents.length; i += 4) {
+            const docs = args.documents.slice(i, i + 4);
+            const inputs = tokenizer(
+              docs.map(() => args.query),
+              {
+                text_pair: docs,
+                padding: true,
+                truncation: true,
+                max_length: 512,
+              },
+            );
+            const { logits } = await ranker(inputs);
+            value.push(
+              ...Array.from(logits.data as Float32Array).map(
+                (n) => 1 / (1 + Math.exp(-n)),
+              ),
+            );
+          }
+        } else throw new Error("未知推理方法");
+        parentPort.postMessage({ id, value });
+      } catch (e) {
+        parentPort.postMessage({ id, error: e.message });
+      }
+    })
+    .catch((e) => parentPort.postMessage({ id: message.id, error: e.message }));
+});

package/src/mcp.ts

@@ -0,0 +1,112 @@
+import type { ConfigSource, AvailableTool } from "./types.js";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
+export class McpManager {
+  config: ConfigSource;
+  connections: Map<string, Promise<Client>>;
+  closed: boolean;
+
+  constructor(config: ConfigSource) {
+    this.config = config;
+    this.connections = new Map();
+    this.closed = false;
+  }
+  async connect(name) {
+    if (this.closed) throw new Error("MCP 已关闭");
+    const spec = this.config().mcp.find((x) => x.name === name && x.enabled);
+    if (!spec) throw new Error("MCP 服务未启用");
+    if (!this.connections.has(name))
+      this.connections.set(
+        name,
+        (async () => {
+          const client = new Client({ name: "dsh-synapse", version: "0.1.0" });
+          const transport =
+            spec.transport === "stdio"
+              ? new StdioClientTransport({
+                  command: spec.command,
+                  args: JSON.parse(spec.argsJson),
+                  env: {
+                    PATH: process.env.PATH ?? "",
+                    ...(process.env.SystemRoot
+                      ? { SystemRoot: process.env.SystemRoot }
+                      : {}),
+                    ...JSON.parse(spec.envJson),
+                  },
+                  stderr: "ignore",
+                })
+              : spec.transport === "sse"
+                ? new SSEClientTransport(new URL(spec.url), {
+                    requestInit: { headers: JSON.parse(spec.headersJson) },
+                  })
+                : new StreamableHTTPClientTransport(new URL(spec.url), {
+                    requestInit: { headers: JSON.parse(spec.headersJson) },
+                  });
+          try {
+            await client.connect(transport, { timeout: 15000 });
+            if (this.closed) {
+              await client.close();
+              throw new Error("MCP 已关闭");
+            }
+            return client;
+          } catch (e) {
+            await transport.close().catch(() => {});
+            this.connections.delete(name);
+            throw e;
+          }
+        })(),
+      );
+    return this.connections.get(name);
+  }
+  async list(name) {
+    const c = await this.connect(name);
+    let cursor;
+    const tools = [];
+    do {
+      const result = await c.listTools(cursor ? { cursor } : undefined, {
+        timeout: 15000,
+      });
+      tools.push(...result.tools);
+      cursor = result.nextCursor;
+      if (tools.length > 2000) throw new Error("MCP 工具数量超限");
+    } while (cursor);
+    return tools;
+  }
+  async available(): Promise<AvailableTool[]> {
+    const result: AvailableTool[] = [];
+    for (const spec of this.config().mcp.filter((s) => s.enabled)) {
+      const allowed = new Set(spec.allowedTools.split(/\s+/).filter(Boolean));
+      if (!allowed.size) continue;
+      for (const tool of await this.list(spec.name))
+        if (allowed.has(tool.name))
+          result.push({
+            server: spec.name,
+            rawName: tool.name,
+            name: `mcp_${result.length}`,
+            description: `${spec.name}/${tool.name}: ${tool.description || ""}`,
+            parameters: tool.inputSchema,
+          });
+    }
+    return result;
+  }
+  async call(tool, args) {
+    const spec = this.config().mcp.find(
+      (s) => s.name === tool.server && s.enabled,
+    );
+    if (!spec?.allowedTools.split(/\s+/).includes(tool.rawName))
+      throw new Error("工具不在允许列表");
+    return (await this.connect(tool.server)).callTool(
+      { name: tool.rawName, arguments: args },
+      undefined,
+      { timeout: 60000 },
+    );
+  }
+  async close() {
+    this.closed = true;
+    await Promise.allSettled(
+      [...this.connections.values()].map(async (p) => (await p).close()),
+    );
+    this.connections.clear();
+  }
+}

package/src/memory.ts

@@ -0,0 +1,178 @@
+import type {
+  ConfigSource,
+  Embeddings,
+  MemoryRow,
+  MemoryItem,
+} from "./types.js";
+import Database from "better-sqlite3";
+import * as vec from "sqlite-vec";
+import { createHash } from "node:crypto";
+const channelCheck = (c) => {
+  if (!["chat", "review"].includes(c)) throw new Error("记忆分类无效");
+  return c;
+};
+// Whitespace-separated Han characters let unicode61 index Chinese without an external tokenizer.
+export const tokenize = (text: string): string =>
+  (text.toLowerCase().match(/[\p{Script=Han}]|[\p{L}\p{N}_]+/gu) || []).join(
+    " ",
+  );
+export function parseHermes(markdown, channel = "chat") {
+  channelCheck(channel);
+  if (typeof markdown !== "string" || markdown.length > 2000000)
+    throw new Error("记忆文档无效或超过 2 MB");
+  const chunks = [];
+  let heading = "Hermes memory",
+    lines = [];
+  const flush = () => {
+    const content = lines.join("\n").trim();
+    if (content)
+      for (let i = 0; i < content.length; i += 1200)
+        chunks.push({
+          channel,
+          source: "hermes",
+          text: `${heading}\n${content.slice(i, i + 1200)}`,
+        });
+    lines = [];
+  };
+  const clean = markdown.replace(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/, "");
+  let fenced = false;
+  for (const line of clean.split(/\r?\n/)) {
+    if (/^\s*(```|~~~)/.test(line)) fenced = !fenced;
+    const match = !fenced && line.match(/^#{1,6}\s+(.+)/);
+    if (match) {
+      flush();
+      heading = match[1];
+    } else lines.push(line);
+  }
+  flush();
+  return chunks;
+}
+export class Memory {
+  inference: Embeddings;
+  config: ConfigSource;
+  db: Database.Database;
+
+  constructor(file: string, inference: Embeddings, config: ConfigSource) {
+    this.inference = inference;
+    this.config = config;
+    this.db = new Database(file);
+    this.db.pragma("journal_mode = WAL");
+    this.db.pragma("busy_timeout = 5000");
+    vec.load(this.db);
+    this.db
+      .exec(`CREATE TABLE IF NOT EXISTS memories(id INTEGER PRIMARY KEY, channel TEXT NOT NULL, source TEXT NOT NULL, text TEXT NOT NULL, hash TEXT NOT NULL UNIQUE, created TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP);
+      CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(tokens);
+      CREATE VIRTUAL TABLE IF NOT EXISTS memory_vec USING vec0(id INTEGER PRIMARY KEY, embedding float[512], channel TEXT partition key);
+      CREATE TABLE IF NOT EXISTS deliveries(id TEXT PRIMARY KEY, event TEXT NOT NULL, payload TEXT NOT NULL, state TEXT NOT NULL DEFAULT 'pending', attempts INTEGER NOT NULL DEFAULT 0, error TEXT, result TEXT, updated TEXT DEFAULT CURRENT_TIMESTAMP);`);
+  }
+  async addMany(items: MemoryItem[]) {
+    const ready = [];
+    for (const item of items) {
+      channelCheck(item.channel);
+      if (!item.text?.trim()) continue;
+      const hash = createHash("sha256")
+        .update(item.channel + "\0" + item.text)
+        .digest("hex");
+      if (this.db.prepare("SELECT 1 FROM memories WHERE hash=?").get(hash))
+        continue;
+      const embedding = await this.inference.embed(item.text);
+      if (
+        embedding.length !== 512 ||
+        embedding.some((n) => !Number.isFinite(n))
+      )
+        throw new Error("嵌入维度或数值无效");
+      ready.push({ ...item, hash, embedding: Float32Array.from(embedding) });
+    }
+    return this.db.transaction(() => {
+      let count = 0;
+      for (const item of ready) {
+        const result = this.db
+          .prepare(
+            "INSERT OR IGNORE INTO memories(channel,source,text,hash) VALUES(?,?,?,?)",
+          )
+          .run(item.channel, item.source, item.text, item.hash);
+        if (!result.changes) continue;
+        const id = BigInt(result.lastInsertRowid);
+        this.db
+          .prepare("INSERT INTO memory_fts(rowid,tokens) VALUES(?,?)")
+          .run(id, tokenize(item.text));
+        this.db
+          .prepare("INSERT INTO memory_vec(id,embedding,channel) VALUES(?,?,?)")
+          .run(id, item.embedding, item.channel);
+        count++;
+      }
+      const old = this.db
+        .prepare("SELECT id FROM memories ORDER BY id DESC LIMIT -1 OFFSET ?")
+        .all(this.config().maxMemories) as { id: number }[];
+      for (const { id } of old) this.remove(id);
+      return { inserted: count, duplicates: items.length - count };
+    })();
+  }
+  remove(id) {
+    this.db.prepare("DELETE FROM memory_vec WHERE id=?").run(BigInt(id));
+    this.db.prepare("DELETE FROM memory_fts WHERE rowid=?").run(id);
+    this.db.prepare("DELETE FROM memories WHERE id=?").run(id);
+  }
+  async search(query, channel) {
+    channelCheck(channel);
+    const c = this.config();
+    if (!c.ragEnabled) return [];
+    const embedding = Float32Array.from(
+      await this.inference.embed(query, true),
+    );
+    const vectors = this.db
+      .prepare(
+        "SELECT id,distance FROM memory_vec WHERE embedding MATCH ? AND k=? AND channel=? ORDER BY distance",
+      )
+      .all(embedding, c.recallK, channel) as { id: number }[];
+    const terms = [
+      ...new Set(tokenize(query).split(" ").filter(Boolean)),
+    ].slice(0, 64);
+    const lexical = terms.length
+      ? this.db
+          .prepare(
+            "SELECT f.rowid AS id FROM memory_fts f JOIN memories m ON m.id=f.rowid WHERE memory_fts MATCH ? AND m.channel=? ORDER BY rank LIMIT ?",
+          )
+          .all(
+            terms.map((t) => '"' + t.replaceAll('"', '""') + '"').join(" OR "),
+            channel,
+            c.recallK,
+          )
+      : [];
+    const scores = new Map();
+    for (const rows of [vectors, lexical] as { id: number }[][])
+      rows.forEach(({ id }, i) =>
+        scores.set(id, (scores.get(id) || 0) + 1 / (60 + i + 1)),
+      );
+    let results = [...scores]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, c.recallK)
+      .map(([id, score]) => ({
+        ...(this.db
+          .prepare("SELECT * FROM memories WHERE id=?")
+          .get(id) as MemoryRow),
+        score,
+      }));
+    if (c.reranker && results.length) {
+      const ranks = await this.inference.rerank(
+        query,
+        results.map((x) => x.text),
+      );
+      results = results
+        .map((x, i) => ({ ...x, score: ranks[i] }))
+        .sort((a, b) => b.score - a.score);
+    }
+    return results.slice(0, c.memoryK);
+  }
+  list(channel, limit = 100) {
+    channelCheck(channel);
+    return this.db
+      .prepare(
+        "SELECT * FROM memories WHERE channel=? ORDER BY id DESC LIMIT ?",
+      )
+      .all(channel, limit);
+  }
+  close() {
+    this.db.close();
+  }
+}