@minhpnq1807/contextos 0.1.0

package/plugins/ctx/lib/analyzer.js

@@ -0,0 +1,321 @@
+import fs from "node:fs";
+import path from "node:path";
+import { findGraphRelevantFiles, mergeRelevantFiles } from "./graph-retriever.js";
+import { expandImportGraph } from "./import-graph.js";
+import { findEmbeddingRelevantFiles } from "./file-embedding-retriever.js";
+
+const STOP_WORDS = new Set([
+  "a", "an", "and", "are", "as", "at", "be", "by", "cho", "co", "cua", "do", "fix", "for",
+  "from", "in", "is", "it", "la", "of", "on", "or", "sua", "task", "the", "to", "trong",
+  "tra", "va", "with"
+]);
+
+const IMPORTANT_WORDS = [
+  "always", "never", "must", "required", "important", "strictly", "mandatory",
+  "luon", "khong bao gio", "bat buoc", "quan trong"
+];
+
+const IGNORE_DIRS = new Set([
+  ".git", ".next", ".turbo", "coverage", "dist", "build", "node_modules", "vendor"
+]);
+
+const SEMANTIC_ALIASES = {
+  duyet: ["moderation", "moderate", "review", "approve", "approval", "approved", "reject", "rejected"],
+  kiem: ["check", "verify", "validation", "validate"],
+  "kiem-duyet": ["moderation", "moderate", "review", "approve", "approval", "reject"],
+  kiemduyet: ["moderation", "moderate", "review", "approve", "approval", "reject"],
+  moderation: ["duyet", "kiemduyet", "review", "approval", "reject"],
+  moderate: ["duyet", "kiemduyet", "review", "approval", "reject"],
+  review: ["duyet", "moderation", "moderate"],
+  approve: ["duyet", "approval", "approved"],
+  approval: ["duyet", "approve", "approved"],
+  reject: ["duyet", "rejected", "rejection"],
+  flow: ["workflow", "pipeline", "process"],
+  workflow: ["flow", "pipeline", "process"],
+  tai: ["upload", "uploaded", "resource"],
+  "tai-len": ["upload", "uploaded", "resource"],
+  tailen: ["upload", "uploaded", "resource"],
+  upload: ["tai", "tailen", "resource", "uploaded"],
+  xac: ["confirm", "verify", "verification"],
+  nhan: ["confirm", "confirmation"],
+  "xac-nhan": ["confirm", "confirmation", "verify", "verification"],
+  xacnhan: ["confirm", "confirmation", "verify", "verification"],
+  thong: ["notification", "notify", "message"],
+  bao: ["notification", "notify", "message"],
+  "thong-bao": ["notification", "notify", "message"],
+  thongbao: ["notification", "notify", "message"]
+};
+
+const MODERATION_TOKENS = new Set(["moderation", "moderate", "content-moderation", "approval", "approved", "reject", "rejected", "needs_review"]);
+
+export function tokenize(value) {
+  const normalized = String(value || "")
+    .toLowerCase()
+    .normalize("NFD")
+    .replace(/[\u0300-\u036f]/g, "")
+    .replace(/kiem\s+duyet/g, "kiem-duyet")
+    .replace(/tai\s+len/g, "tai-len")
+    .replace(/xac\s+nhan/g, "xac-nhan")
+    .replace(/thong\s+bao/g, "thong-bao");
+
+  return normalized
+    .split(/[^a-z0-9_.-]+/g)
+    .flatMap(splitCompoundToken)
+    .filter((word) => word.length > 1 && !STOP_WORDS.has(word));
+}
+
+function splitCompoundToken(token) {
+  const parts = String(token || "").split(/[_.-]+/g).filter(Boolean);
+  return parts.length > 1 ? [token, ...parts] : [token];
+}
+
+function expandSemanticTokens(tokens) {
+  const expanded = new Set(tokens);
+  for (const token of tokens) {
+    for (const alias of SEMANTIC_ALIASES[token] || []) expanded.add(alias);
+  }
+  return expanded;
+}
+
+function sourceFromLine(line) {
+  const match = line.match(/^## Source:\s+(.+)$/);
+  return match ? match[1].trim() : null;
+}
+
+function cleanRuleLine(line) {
+  return line
+    .replace(/^\s{0,3}[-*+]\s+/, "")
+    .replace(/^\s{0,3}\d+[.)]\s+/, "")
+    .replace(/^#+\s+/, "")
+    .trim();
+}
+
+export function parseRules(markdown) {
+  const rules = [];
+  let sourcePath = "unknown";
+  let paragraph = [];
+
+  const flushParagraph = () => {
+    const content = cleanRuleLine(paragraph.join(" ").replace(/\s+/g, " "));
+    paragraph = [];
+    if (content.length < 20) return;
+    rules.push({
+      id: `r${rules.length + 1}`,
+      sourcePath,
+      content,
+      originalOrder: rules.length
+    });
+  };
+
+  for (const rawLine of String(markdown || "").split(/\r?\n/)) {
+    const line = rawLine.trim();
+    const nextSource = sourceFromLine(line);
+    if (nextSource) {
+      flushParagraph();
+      sourcePath = nextSource;
+      continue;
+    }
+    if (!line || /^-{3,}$/.test(line)) {
+      flushParagraph();
+      continue;
+    }
+    if (/^\s{0,3}([-*+]|\d+[.)])\s+/.test(rawLine) || /^#{1,6}\s+/.test(rawLine)) {
+      flushParagraph();
+      const content = cleanRuleLine(rawLine);
+      if (content.length >= 4) {
+        rules.push({
+          id: `r${rules.length + 1}`,
+          sourcePath,
+          content,
+          originalOrder: rules.length
+        });
+      }
+      continue;
+    }
+    paragraph.push(line);
+  }
+  flushParagraph();
+  return dedupeRules(rules);
+}
+
+function dedupeRules(rules) {
+  const seen = new Set();
+  return rules.filter((rule) => {
+    const key = `${rule.sourcePath}:${rule.content.toLowerCase()}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  }).map((rule, index) => ({ ...rule, id: `r${index + 1}`, originalOrder: index }));
+}
+
+export function scoreRules(rules, task, openFiles = []) {
+  const rawTaskTokens = new Set(tokenize(task));
+  const openFileText = Array.isArray(openFiles) ? openFiles.join(" ") : String(openFiles || "");
+  const openFileTokens = new Set(tokenize(openFileText));
+
+  return rules.map((rule) => {
+    const ruleTokens = new Set(tokenize(rule.content));
+    const exactOverlap = [...rawTaskTokens].filter((token) => ruleTokens.has(token));
+    const semanticOverlap = [];
+    for (const token of rawTaskTokens) {
+      for (const alias of SEMANTIC_ALIASES[token] || []) {
+        if (!rawTaskTokens.has(alias) && ruleTokens.has(alias)) semanticOverlap.push(`${token}->${alias}`);
+      }
+    }
+    const reasons = [];
+    let score = rawTaskTokens.size
+      ? (exactOverlap.length + semanticOverlap.length * 0.5) / Math.max(rawTaskTokens.size, 1)
+      : 0;
+
+    if (exactOverlap.length) reasons.push(`task:${exactOverlap.join("/")}`);
+    if (semanticOverlap.length) reasons.push(`semantic:${semanticOverlap.join("/")}`);
+
+    const lowerRule = rule.content.toLowerCase();
+    if (IMPORTANT_WORDS.some((word) => lowerRule.includes(word))) {
+      score += 0.4;
+      reasons.push("imperative");
+    }
+
+    const fileMentions = [...ruleTokens].filter((token) => /[./]/.test(token) || /\.[a-z0-9]+$/.test(token));
+    if (fileMentions.some((token) => openFileTokens.has(token) || openFileText.includes(token))) {
+      score += 0.2;
+      reasons.push("open-file");
+    }
+
+    return {
+      ...rule,
+      score: Math.max(0, Math.min(1, Number(score.toFixed(3)))),
+      reasons
+    };
+  }).sort((a, b) => b.score - a.score || a.originalOrder - b.originalOrder);
+}
+
+export async function findRelevantFiles({
+  cwd = process.cwd(),
+  task = "",
+  rules = [],
+  dataDir,
+  limit = 3,
+  embeddingFileFinder = findEmbeddingRelevantFiles,
+  fileEmbeddingTimeoutMs,
+  fileEmbeddingOptions = {}
+} = {}) {
+  const rawTaskTokens = new Set(tokenize(task));
+  if (!rawTaskTokens.size) return [];
+
+  const candidates = [];
+  walkFiles(cwd, (filePath) => {
+    const rel = path.relative(cwd, filePath);
+    const fileTokens = new Set(tokenize(rel));
+    const match = scoreFileTokens({ rawTaskTokens, fileTokens });
+    if (match.score > 0) {
+      candidates.push({
+        path: rel,
+        score: match.score,
+        reasons: match.reasons
+      });
+    }
+  });
+
+  const heuristicFiles = candidates
+    .sort((a, b) => b.score - a.score || a.path.localeCompare(b.path))
+    .slice(0, Math.max(limit * 2, 6));
+  const hasHighConfidenceHeuristics =
+    heuristicFiles.length >= limit &&
+    Number(heuristicFiles[0]?.score || 0) >= 8;
+  const embeddingFiles = hasHighConfidenceHeuristics
+    ? []
+    : await embeddingFileFinder({
+      cwd,
+      task,
+      dataDir,
+      timeoutMs: fileEmbeddingTimeoutMs,
+      embeddingOptions: fileEmbeddingOptions,
+      limit: Math.max(limit * 2, 6)
+    });
+  const importGraphFiles = expandImportGraph({
+    cwd,
+    seedFiles: mergeLocalFileCandidates([...heuristicFiles, ...embeddingFiles]).slice(0, limit),
+    limit: Math.max(limit * 2, 6)
+  });
+  const seedFiles = mergeLocalFileCandidates([...heuristicFiles, ...embeddingFiles, ...importGraphFiles])
+    .slice(0, Math.max(limit * 3, 9));
+
+  const graphFiles = findGraphRelevantFiles({
+    cwd,
+    task,
+    rules,
+    seedFiles,
+    limit: Math.max(limit * 2, 6)
+  });
+
+  return mergeRelevantFiles({ graphFiles, heuristicFiles: seedFiles, limit });
+}
+
+function mergeLocalFileCandidates(files) {
+  const byPath = new Map();
+  for (const file of files) {
+    const existing = byPath.get(file.path);
+    byPath.set(file.path, {
+      ...existing,
+      ...file,
+      score: Number(existing?.score || 0) + Number(file.score || 0),
+      reasons: [...new Set([...(existing?.reasons || []), ...(file.reasons || [])])],
+      source: existing?.source === "import-graph" || file.source === "import-graph" ? "import-graph" : file.source
+    });
+  }
+  return [...byPath.values()].sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+}
+
+function scoreFileTokens({ rawTaskTokens, fileTokens }) {
+  let score = 0;
+  const reasons = new Set();
+  const hasModerationIntent = rawTaskTokens.has("kiem-duyet") || rawTaskTokens.has("kiemduyet") || rawTaskTokens.has("duyet");
+  const hasUploadIntent = rawTaskTokens.has("upload") || rawTaskTokens.has("tai-len") || rawTaskTokens.has("tailen");
+
+  for (const token of rawTaskTokens) {
+    if (fileTokens.has(token)) {
+      score += 3;
+      reasons.add(token);
+    }
+    for (const alias of SEMANTIC_ALIASES[token] || []) {
+      if (fileTokens.has(alias)) {
+        score += 2;
+        reasons.add(`${token}->${alias}`);
+      }
+    }
+  }
+
+  if (hasModerationIntent && [...fileTokens].some((token) => MODERATION_TOKENS.has(token))) {
+    score += 6;
+    reasons.add("domain:moderation");
+  }
+
+  if (hasUploadIntent && (fileTokens.has("upload") || fileTokens.has("uploaded") || fileTokens.has("resource"))) {
+    score += 2;
+    reasons.add("domain:upload");
+  }
+
+  return { score, reasons: [...reasons] };
+}
+
+function walkFiles(directory, onFile, depth = 0) {
+  if (depth > 6) return;
+  let entries = [];
+  try {
+    entries = fs.readdirSync(directory, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    if (entry.name.startsWith(".") && entry.name !== ".github") {
+      if (entry.name !== ".codex") continue;
+    }
+    const fullPath = path.join(directory, entry.name);
+    if (entry.isDirectory()) {
+      if (!IGNORE_DIRS.has(entry.name)) walkFiles(fullPath, onFile, depth + 1);
+    } else if (entry.isFile()) {
+      onFile(fullPath);
+    }
+  }
+}

package/plugins/ctx/lib/ctx-mcp-client.js

@@ -0,0 +1,52 @@
+import fs from "node:fs";
+import net from "node:net";
+import os from "node:os";
+import path from "node:path";
+
+const DEFAULT_TIMEOUT_MS = 1000;
+
+export function ctxMcpSocketPath(dataDir = defaultDataDir()) {
+  return path.join(dataDir, "ctx-mcp.sock");
+}
+
+export async function callCtxScoreContext(payload, {
+  dataDir = defaultDataDir(),
+  timeoutMs = Number(process.env.CONTEXTOS_MCP_BRIDGE_TIMEOUT_MS || DEFAULT_TIMEOUT_MS)
+} = {}) {
+  const socketPath = ctxMcpSocketPath(dataDir);
+  if (!fs.existsSync(socketPath)) {
+    throw new Error(`ctx-mcp bridge socket not found: ${socketPath}`);
+  }
+
+  return new Promise((resolve, reject) => {
+    const client = net.createConnection(socketPath);
+    let raw = "";
+    const timer = setTimeout(() => {
+      client.destroy();
+      reject(new Error(`ctx-mcp bridge timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+
+    client.on("connect", () => {
+      client.write(`${JSON.stringify(payload)}\n`);
+    });
+    client.on("data", (chunk) => {
+      raw += chunk.toString("utf8");
+    });
+    client.on("end", () => {
+      clearTimeout(timer);
+      try {
+        resolve(JSON.parse(raw || "{}"));
+      } catch (error) {
+        reject(error);
+      }
+    });
+    client.on("error", (error) => {
+      clearTimeout(timer);
+      reject(error);
+    });
+  });
+}
+
+function defaultDataDir() {
+  return process.env.PLUGIN_DATA || path.join(process.env.CODEX_HOME || path.join(os.homedir(), ".codex"), "contextos");
+}

package/plugins/ctx/lib/embedding-scorer.js

@@ -0,0 +1,248 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const DEFAULT_MODEL = "Xenova/all-MiniLM-L6-v2";
+const DEFAULT_TIMEOUT_MS = 800;
+const SEMANTIC_HIGH_THRESHOLD = 0.5;
+
+const extractorPromises = new Map();
+let sqlPromise = null;
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(__dirname, "..", "..", "..");
+
+export async function enhanceRuleScoresWithEmbeddings(
+  rules,
+  task,
+  {
+    dataDir = path.join(os.homedir(), ".codex", "contextos"),
+    sources = [],
+    timeoutMs = Number(process.env.CONTEXTOS_EMBEDDING_TIMEOUT_MS || DEFAULT_TIMEOUT_MS),
+    allowRemote = process.env.CONTEXTOS_EMBEDDING_ALLOW_REMOTE === "1",
+    enabled = process.env.CONTEXTOS_EMBEDDINGS !== "0"
+  } = {}
+) {
+  if (!enabled || !String(task || "").trim() || !rules?.length) {
+    return { rules, status: "disabled" };
+  }
+  const cachePath = path.join(dataDir, "embeddings.db");
+  if (!allowRemote && !fs.existsSync(cachePath)) {
+    return { rules, status: "cold-cache", cachePath };
+  }
+
+  try {
+    return await withTimeout(
+      enhanceRuleScores(rules, task, { dataDir, sources, allowRemote }),
+      timeoutMs
+    );
+  } catch (error) {
+    return {
+      rules,
+      status: "fallback",
+      error: error?.message || String(error)
+    };
+  }
+}
+
+export async function warmRuleEmbeddings({
+  rules = [],
+  task = "",
+  dataDir = path.join(os.homedir(), ".codex", "contextos"),
+  sources = [],
+  allowRemote = true
+} = {}) {
+  const texts = [...new Set([
+    task,
+    ...rules.map((rule) => rule.content || "")
+  ].filter((text) => String(text).trim()))];
+
+  const cache = await openEmbeddingCache(dataDir);
+  const embedder = await getExtractor({ allowRemote, dataDir });
+  for (const text of texts) {
+    await getCachedEmbedding({ cache, embedder, text, sources });
+  }
+  cache.close();
+  return { count: texts.length, cachePath: cache.path };
+}
+
+async function enhanceRuleScores(rules, task, { dataDir, sources, allowRemote }) {
+  const cache = await openEmbeddingCache(dataDir);
+  const embedder = await getExtractor({ allowRemote, dataDir });
+  const taskEmbedding = await getCachedEmbedding({ cache, embedder, text: task, sources });
+
+  const enhanced = [];
+  for (const rule of rules) {
+    const ruleEmbedding = await getCachedEmbedding({
+      cache,
+      embedder,
+      text: rule.content || "",
+      sources
+    });
+    const similarity = cosine(taskEmbedding, ruleEmbedding);
+    const semanticScore = similarityToScore(similarity);
+    const baseScore = Number(rule.score || 0);
+    const score = semanticScore >= SEMANTIC_HIGH_THRESHOLD
+      ? Math.max(baseScore, semanticScore)
+      : baseScore;
+
+    enhanced.push({
+      ...rule,
+      score: Math.max(0, Math.min(1, Number(score.toFixed(3)))),
+      embeddingScore: Number(semanticScore.toFixed(3)),
+      reasons: semanticScore >= 0.45
+        ? [...new Set([...(rule.reasons || []), `embedding:${semanticScore.toFixed(2)}`])]
+        : rule.reasons
+    });
+  }
+
+  cache.close();
+  return {
+    rules: enhanced.sort((a, b) => b.score - a.score || a.originalOrder - b.originalOrder),
+    status: "enabled",
+    model: DEFAULT_MODEL,
+    cachePath: cache.path
+  };
+}
+
+async function getExtractor({ allowRemote, dataDir }) {
+  const cacheDir = modelCacheDir(dataDir);
+  const key = `${allowRemote ? "remote" : "local"}:${cacheDir}`;
+  if (!extractorPromises.has(key)) {
+    extractorPromises.set(key, (async () => {
+      const transformers = await import("@xenova/transformers");
+      transformers.env.allowRemoteModels = Boolean(allowRemote);
+      transformers.env.allowLocalModels = true;
+      transformers.env.cacheDir = cacheDir;
+      return transformers.pipeline("feature-extraction", DEFAULT_MODEL, {
+        quantized: true
+      });
+    })());
+  }
+  return extractorPromises.get(key);
+}
+
+export function modelCacheDir(dataDir = path.join(os.homedir(), ".codex", "contextos")) {
+  return path.join(dataDir, "models");
+}
+
+async function getCachedEmbedding({ cache, embedder, text, sources }) {
+  const key = cacheKey(text, sources);
+  const existing = cache.get(key);
+  if (existing) return existing;
+
+  const output = await embedder(String(text || ""), {
+    pooling: "mean",
+    normalize: true
+  });
+  const embedding = Array.from(output.data || []);
+  cache.set(key, embedding);
+  return embedding;
+}
+
+async function openEmbeddingCache(dataDir) {
+  fs.mkdirSync(dataDir, { recursive: true });
+  const cachePath = path.join(dataDir, "embeddings.db");
+  const SQL = await getSql();
+  const buffer = fs.existsSync(cachePath) ? fs.readFileSync(cachePath) : null;
+  const db = buffer?.length ? new SQL.Database(buffer) : new SQL.Database();
+
+  db.run(`
+    CREATE TABLE IF NOT EXISTS embeddings (
+      key TEXT PRIMARY KEY,
+      model TEXT NOT NULL,
+      vector TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    )
+  `);
+
+  return {
+    path: cachePath,
+    get(key) {
+      const stmt = db.prepare("SELECT vector FROM embeddings WHERE key = ? AND model = ?");
+      try {
+        stmt.bind([key, DEFAULT_MODEL]);
+        if (!stmt.step()) return null;
+        return JSON.parse(stmt.getAsObject().vector);
+      } finally {
+        stmt.free();
+      }
+    },
+    set(key, vector) {
+      db.run(
+        "INSERT OR REPLACE INTO embeddings (key, model, vector, updated_at) VALUES (?, ?, ?, ?)",
+        [key, DEFAULT_MODEL, JSON.stringify(vector), new Date().toISOString()]
+      );
+      fs.writeFileSync(cachePath, Buffer.from(db.export()));
+    },
+    close() {
+      fs.writeFileSync(cachePath, Buffer.from(db.export()));
+      db.close();
+    }
+  };
+}
+
+async function getSql() {
+  if (!sqlPromise) {
+    sqlPromise = (async () => {
+      const initSqlJs = (await import("sql.js")).default;
+      return initSqlJs({
+        locateFile: (file) => path.join(repoRoot, "node_modules", "sql.js", "dist", file)
+      });
+    })();
+  }
+  return sqlPromise;
+}
+
+function cacheKey(text, sources) {
+  return crypto
+    .createHash("sha256")
+    .update(DEFAULT_MODEL)
+    .update("\0")
+    .update(String(text || ""))
+    .update("\0")
+    .update(sourceFingerprint(sources))
+    .digest("hex");
+}
+
+function sourceFingerprint(sources) {
+  const parts = [];
+  for (const source of sources || []) {
+    try {
+      const stat = fs.statSync(source);
+      parts.push(`${source}:${stat.mtimeMs}:${stat.size}`);
+    } catch {
+      parts.push(String(source));
+    }
+  }
+  return parts.join("|");
+}
+
+function cosine(a, b) {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  const length = Math.min(a?.length || 0, b?.length || 0);
+  for (let index = 0; index < length; index += 1) {
+    dot += a[index] * b[index];
+    normA += a[index] * a[index];
+    normB += b[index] * b[index];
+  }
+  if (!normA || !normB) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+function similarityToScore(similarity) {
+  return Math.max(0, Math.min(1, (similarity + 1) / 2));
+}
+
+function withTimeout(promise, timeoutMs) {
+  return Promise.race([
+    promise,
+    new Promise((_, reject) => {
+      setTimeout(() => reject(new Error(`embedding scorer timed out after ${timeoutMs}ms`)), timeoutMs);
+    })
+  ]);
+}