@minhpnq1807/contextos 0.5.53 → 0.6.1

package/eval/hallucination/run-leaderboard.js

@@ -0,0 +1,183 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { parseEvalYaml, runSkillRoutingEval } from "../skill-routing/run-eval.js";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const evalRoot = path.resolve(__dirname, "..", "skill-routing");
+const DEFAULT_CASE_LIMIT = 20;
+
+export async function runHallucinationLeaderboard({
+  rootDir = path.resolve(__dirname, "..", ".."),
+  casesPath = path.join(evalRoot, "cases.yaml"),
+  caseLimit = DEFAULT_CASE_LIMIT
+} = {}) {
+  const config = parseEvalYaml(fs.readFileSync(casesPath, "utf8"));
+  const selectedCases = selectLeaderboardCases(config.cases, caseLimit);
+  const wanted = new Set(selectedCases.map(caseId));
+  const contextos = await runSkillRoutingEval({ rootDir, casesPath, topK: 3, threshold: 0.5 });
+  const contextRows = contextos.rows
+    .filter((row) => wanted.has(caseId(row)))
+    .map((row) => evaluateRow({
+      prompt: row.prompt,
+      fixture: row.fixture,
+      expected: row.expected,
+      allowed: row.allowed,
+      forbidden: row.forbidden,
+      selectedIds: row.selectedIds
+    }));
+  const rawRows = selectedCases.map((testCase) => evaluateRow({
+    prompt: testCase.prompt,
+    fixture: testCase.fixture,
+    expected: testCase.expected || [],
+    allowed: testCase.allowed || [],
+    forbidden: testCase.forbidden || [],
+    selectedIds: rawAgentSkills({ prompt: testCase.prompt, skills: config.skills, topK: 3 })
+  }));
+
+  return {
+    caseCount: selectedCases.length,
+    repoCount: new Set(selectedCases.map((row) => row.fixture)).size,
+    systems: [
+      summarizeSystem("Raw Agent", rawRows),
+      summarizeSystem("ContextOS + Codex", contextRows)
+    ],
+    rows: selectedCases.map((testCase) => ({
+      prompt: testCase.prompt,
+      fixture: testCase.fixture,
+      expected: testCase.expected || [],
+      raw: rawRows.find((row) => row.prompt === testCase.prompt && row.fixture === testCase.fixture),
+      contextos: contextRows.find((row) => row.prompt === testCase.prompt && row.fixture === testCase.fixture)
+    }))
+  };
+}
+
+export function formatHallucinationLeaderboard(result) {
+  const lines = [
+    "Hallucination Leaderboard",
+    `Repos: ${result.repoCount}`,
+    `Tasks: ${result.caseCount}`,
+    "",
+    "System              Correct Skill",
+    "------------------  -------------"
+  ];
+  for (const system of result.systems) {
+    lines.push(`${system.name.padEnd(18)}  ${percent(system.correctRate)}`);
+  }
+  lines.push("", "Sample failures:");
+  const failures = result.rows
+    .filter((row) => !row.raw.correct || !row.contextos.correct)
+    .slice(0, 6);
+  if (!failures.length) {
+    lines.push("- none");
+  } else {
+    for (const row of failures) {
+      lines.push(`- ${row.fixture}: "${row.prompt}"`);
+      lines.push(`  expected: ${row.expected.join(", ") || "(none)"}`);
+      lines.push(`  raw: ${row.raw.selectedIds.join(", ") || "(none)"} ${row.raw.correct ? "✓" : "✗"}`);
+      lines.push(`  contextos: ${row.contextos.selectedIds.join(", ") || "(none)"} ${row.contextos.correct ? "✓" : "✗"}`);
+    }
+  }
+  return lines.join("\n");
+}
+
+function selectLeaderboardCases(cases, limit) {
+  const wantedFixtures = [
+    "expo-eas",
+    "next-vercel",
+    "docker-node",
+    "railway-render",
+    "firebase-hosting",
+    "nest-prisma",
+    "express-mongo-jwt",
+    "oauth-google",
+    "redis-cache",
+    "contextos",
+    "frontend-only-next",
+    "static-docs"
+  ];
+  const selected = [];
+  for (const fixture of wantedFixtures) {
+    const match = cases.find((row) => row.fixture === fixture && !selected.some((item) => caseId(item) === caseId(row)));
+    if (match) selected.push(match);
+    if (selected.length >= limit) return selected;
+  }
+  for (const row of cases) {
+    if (!selected.some((item) => caseId(item) === caseId(row))) selected.push(row);
+    if (selected.length >= limit) break;
+  }
+  return selected;
+}
+
+function rawAgentSkills({ prompt, skills, topK }) {
+  return skills
+    .map((skill) => ({ id: skill.id, score: rawPromptScore(prompt, skill) }))
+    .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id))
+    .slice(0, topK)
+    .map((skill) => skill.id);
+}
+
+function rawPromptScore(prompt, skill) {
+  const promptTokens = new Set(tokenize(prompt));
+  const triggerTokens = tokenize([
+    skill.id,
+    skill.description,
+    ...(skill.positive_triggers?.prompts || [])
+  ].join(" "));
+  let score = 0;
+  for (const token of triggerTokens) {
+    if (promptTokens.has(token)) score += token.length > 5 ? 2 : 1;
+  }
+  return score;
+}
+
+function evaluateRow({ prompt, fixture, expected, allowed, forbidden, selectedIds }) {
+  const selected = new Set(selectedIds);
+  const accepted = new Set([...expected, ...allowed]);
+  const hasExpected = expected.length
+    ? expected.every((skill) => selected.has(skill))
+    : selectedIds.length === 0;
+  const hasForbidden = forbidden.some((skill) => selected.has(skill));
+  const hasUnexpected = selectedIds.some((skill) => !accepted.has(skill));
+  return {
+    prompt,
+    fixture,
+    expected,
+    allowed,
+    forbidden,
+    selectedIds,
+    correct: hasExpected && !hasForbidden && !hasUnexpected
+  };
+}
+
+function summarizeSystem(name, rows) {
+  const correct = rows.filter((row) => row.correct).length;
+  return {
+    name,
+    correct,
+    total: rows.length,
+    correctRate: rows.length ? correct / rows.length : 0
+  };
+}
+
+function tokenize(value) {
+  return String(value || "")
+    .toLowerCase()
+    .split(/[^a-z0-9@.-]+/)
+    .filter((token) => token.length > 2);
+}
+
+function caseId(row) {
+  return `${row.fixture}\0${row.prompt}`;
+}
+
+function percent(value) {
+  return `${(value * 100).toFixed(1)}%`;
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const result = await runHallucinationLeaderboard();
+  console.log(formatHallucinationLeaderboard(result));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minhpnq1807/contextos",
-  "version": "0.5.53",
+  "version": "0.6.1",
   "description": "Task-aware AGENTS.md context injection and compliance reporting for Codex, Claude Code, and Antigravity.",
   "type": "module",
   "bin": {
@@ -10,10 +10,13 @@
   "files": [
     "bin/",
     "plugins/",
+    ".codex/skills/",
+    ".codex/workflows/",
     ".agents/",
     "README.md",
     "DEMO.md",
     "LAUNCH.md",
+    "community-skills/",
     "eval/",
     "docs/",
     "LICENSE",
@@ -24,6 +27,7 @@
     "build": "node bin/ctx.js --version",
     "validate:plugin": "node test/validate-plugin.js",
     "benchmark:skills": "node bin/ctx.js benchmark --skills",
+    "leaderboard:hallucination": "node eval/hallucination/run-leaderboard.js",
     "test:mcp": "node test/mcp-protocol-smoke.js"
   },
   "engines": {

package/plugins/ctx/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "ctx",
-  "version": "0.5.53",
+  "version": "0.6.1",
   "description": "Inject task-relevant AGENTS.md rules into Codex through plugin hooks.",
   "author": {
     "name": "ContextOS"

package/plugins/ctx/lib/certification.js

@@ -0,0 +1,223 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { filterActionableRules, parseRules } from "./analyzer.js";
+import { readAgentsChain } from "./reader.js";
+import { scanSkills } from "./skill-discoverer.js";
+import { scanWorkflows } from "./workflow-discoverer.js";
+
+const PROJECT_SKILL_ROOTS = [
+  [".codex", "skills"],
+  [".agents", "skills"],
+  [".claude", "skills"],
+  [".gemini", "skills"],
+  [".gemini", "antigravity", "skills"],
+  [".gemini", "antigravity-cli", "skills"]
+];
+
+const PROJECT_WORKFLOW_ROOTS = [
+  [".claude", "workflows"],
+  [".codex", "workflows"],
+  [".gemini", "workflows"],
+  [".gemini", "antigravity", "workflows"],
+  [".gemini", "antigravity-cli", "workflows"]
+];
+
+export function inspectContextOSReady({ cwd = process.cwd(), home = os.homedir() } = {}) {
+  const root = findProjectRoot(cwd);
+  const rules = inspectRules({ cwd, root, home });
+  const skills = inspectSkills({ root });
+  const workflows = inspectWorkflows({ root });
+  const overall = Math.round((rules.score + skills.score + workflows.score) / 3);
+  const tier = readinessTier(overall, { rules, skills, workflows });
+
+  return {
+    root,
+    rules,
+    skills,
+    workflows,
+    overall,
+    tier,
+    badge: tier === "Not Ready" ? "ContextOS Ready: Not Ready" : `ContextOS Ready ${tier}`
+  };
+}
+
+export function formatContextOSReady(result) {
+  const lines = [
+    "Repository Score",
+    "",
+    `Rules: ${result.rules.score}`,
+    `Skills: ${result.skills.score}`,
+    `Workflows: ${result.workflows.score}`,
+    "",
+    "Overall:",
+    result.badge,
+    "",
+    "Evidence:",
+    `- Rules: ${result.rules.summary}`,
+    `- Skills: ${result.skills.summary}`,
+    `- Workflows: ${result.workflows.summary}`
+  ];
+
+  const next = [
+    ...result.rules.recommendations,
+    ...result.skills.recommendations,
+    ...result.workflows.recommendations
+  ];
+  if (next.length) {
+    lines.push("", "Next:");
+    for (const item of [...new Set(next)].slice(0, 5)) lines.push(`- ${item}`);
+  }
+
+  return lines.join("\n");
+}
+
+function inspectRules({ cwd, root, home }) {
+  const chain = readAgentsChain({ cwd, home });
+  const projectSources = chain.sources.filter((source) => isInsidePath(source, root));
+  const rules = parseRules(chain.content || "");
+  const actionable = filterActionableRules(rules);
+  const imperative = actionable.filter((rule) => /\b(always|never|must|required|use|prefer|avoid|do not|don't)\b/i.test(rule.content));
+  let score = 0;
+  const recommendations = [];
+
+  if (projectSources.length) score += 55;
+  else recommendations.push("Add a project AGENTS.md with repository-specific operating rules.");
+
+  if (actionable.length >= 3) score += 20;
+  else recommendations.push("Add at least three actionable AGENTS.md rules.");
+
+  if (imperative.length) score += 15;
+  else recommendations.push("Use explicit rule language such as always, never, must, use, prefer, or avoid.");
+
+  if (projectSources.length > 1 || fs.existsSync(path.join(root, ".ruler"))) score += 10;
+
+  return {
+    score: Math.min(100, score),
+    sources: projectSources,
+    ruleCount: rules.length,
+    actionableCount: actionable.length,
+    summary: projectSources.length
+      ? `${projectSources.length} AGENTS.md source(s), ${actionable.length} actionable rule(s)`
+      : "missing project AGENTS.md",
+    recommendations
+  };
+}
+
+function inspectSkills({ root }) {
+  const roots = PROJECT_SKILL_ROOTS.map((parts) => path.join(root, ...parts));
+  const skills = scanSkills({ cwd: root, roots, maxSkills: 500 });
+  const metadataFiles = findFiles(roots, (filePath) => /skill\.ya?ml$/i.test(path.basename(filePath)));
+  const richMetadata = metadataFiles.filter((filePath) => {
+    const content = safeRead(filePath);
+    return /^positive_triggers:/m.test(content)
+      && /^evidence:/m.test(content)
+      && /^negative_triggers:/m.test(content)
+      && /^workflow:/m.test(content);
+  });
+  let score = 0;
+  const recommendations = [];
+
+  if (skills.length) score += 50;
+  else recommendations.push("Add project skills under .codex/skills/ or .agents/skills/.");
+
+  if (metadataFiles.length) score += 20;
+  else recommendations.push("Add skill.yaml metadata beside important SKILL.md files.");
+
+  if (richMetadata.length) score += 20;
+  else recommendations.push("Include positive_triggers, negative_triggers, evidence, and workflow in skill.yaml.");
+
+  if (skills.length >= 3) score += 10;
+  else recommendations.push("Provide at least three project-relevant skills for common tasks.");
+
+  return {
+    score: Math.min(100, score),
+    count: skills.length,
+    metadataCount: metadataFiles.length,
+    richMetadataCount: richMetadata.length,
+    summary: skills.length
+      ? `${skills.length} skill(s), ${metadataFiles.length} metadata file(s)`
+      : "missing project skill packs",
+    recommendations
+  };
+}
+
+function inspectWorkflows({ root }) {
+  const roots = PROJECT_WORKFLOW_ROOTS.map((parts) => path.join(root, ...parts));
+  const workflows = scanWorkflows({ cwd: root, roots });
+  const withChain = workflows.filter((workflow) => workflow.chain?.length);
+  let score = 0;
+  const recommendations = [];
+
+  if (workflows.length) score += 60;
+  else recommendations.push("Add project workflows under .codex/workflows/ or .claude/workflows/.");
+
+  if (withChain.length) score += 25;
+  else recommendations.push("Include agent handoff names such as planner, tester, code-reviewer, or docs-manager in workflow files.");
+
+  if (workflows.length >= 2) score += 15;
+  else recommendations.push("Provide more than one workflow when the repo has distinct delivery paths.");
+
+  return {
+    score: Math.min(100, score),
+    count: workflows.length,
+    chainCount: withChain.length,
+    summary: workflows.length
+      ? `${workflows.length} workflow(s), ${withChain.length} with agent chain(s)`
+      : "missing project workflows",
+    recommendations
+  };
+}
+
+function readinessTier(overall, { rules, skills, workflows }) {
+  if (rules.score < 50 || skills.score < 50 || workflows.score < 50) return "Not Ready";
+  if (overall >= 85) return "Gold";
+  if (overall >= 70) return "Silver";
+  if (overall >= 50) return "Bronze";
+  return "Not Ready";
+}
+
+function findProjectRoot(cwd) {
+  let current = path.resolve(cwd);
+  while (true) {
+    if (fs.existsSync(path.join(current, ".git"))) return current;
+    const parent = path.dirname(current);
+    if (parent === current) return path.resolve(cwd);
+    current = parent;
+  }
+}
+
+function findFiles(roots, predicate) {
+  const files = [];
+  for (const root of roots) walk(root, files, predicate, 0);
+  return files;
+}
+
+function walk(directory, files, predicate, depth) {
+  if (depth > 4) return;
+  let entries = [];
+  try {
+    entries = fs.readdirSync(directory, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const entry of entries) {
+    const filePath = path.join(directory, entry.name);
+    if (entry.isDirectory()) walk(filePath, files, predicate, depth + 1);
+    else if (entry.isFile() && predicate(filePath)) files.push(filePath);
+  }
+}
+
+function isInsidePath(filePath, root) {
+  const relative = path.relative(path.resolve(root), path.resolve(filePath));
+  return relative && !relative.startsWith("..") && !path.isAbsolute(relative);
+}
+
+function safeRead(filePath) {
+  try {
+    return fs.readFileSync(filePath, "utf8");
+  } catch {
+    return "";
+  }
+}