@minhpnq1807/contextos 0.6.1 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.6.2
4
+
5
+ - **Live agent leaderboard:** Added `ctx leaderboard --agents codex,gemini` and `npm run leaderboard:agents` to run the hallucination benchmark through installed Codex/Gemini CLIs with timeouts and skip/error reporting for missing or unauthenticated agents.
6
+ - **Live GIF capture:** Added `npm run demo:capture` to regenerate the three launch GIFs from real local `ctx` command output across ContextOS and skill-routing fixture repos.
7
+ - **Community skills sync:** Added `scripts/sync-community-skills.mjs`, `npm run sync:community-skills`, and a scheduled/manual GitHub Action that opens PRs from `khovan123/contextOS-skills` back into `community-skills/`.
8
+
3
9
  ## 0.6.1
4
10
 
5
11
  - **Hallucination Leaderboard:** Added `ctx leaderboard --hallucination` and `npm run leaderboard:hallucination` to compare raw prompt-only skill guesses against ContextOS evidence-routed skill selection across 20 fixture tasks.
package/README.md CHANGED
@@ -49,6 +49,12 @@ More 10-second demos:
49
49
  | AGENTS.md Lost In The Middle | [docs/demo/agents-lost-middle.gif](docs/demo/agents-lost-middle.gif) |
50
50
  | ContextOS Ready Gold | [docs/demo/contextos-ready.gif](docs/demo/contextos-ready.gif) |
51
51
 
52
+ Regenerate the GIFs from real local `ctx` command output:
53
+
54
+ ```bash
55
+ npm run demo:capture
56
+ ```
57
+
52
58
  ## Agent Hallucination Benchmark
53
59
 
54
60
  Generic agents often guess deployment tooling from the prompt alone:
@@ -267,6 +273,7 @@ The score checks project `AGENTS.md` rules, project skill packs under `.codex/sk
267
273
  | `ctx benchmark -- "task"` | Compare raw AGENTS.md ordering vs ContextOS scheduling. |
268
274
  | `ctx benchmark --skills` | Run the Skill Router eval benchmark. |
269
275
  | `ctx leaderboard --hallucination` | Compare raw prompt-only guesses vs ContextOS routing. |
276
+ | `ctx leaderboard --agents codex,gemini` | Run the live CLI leaderboard when Codex/Gemini credentials are available. |
270
277
  | `ctx sync --rules` | Sync AGENTS/Ruler/MCP config across agents. |
271
278
  | `ctx sync --skills` | Sync skills across agents through skillshare. |
272
279
  | `ctx sync --workflows` | Sync workflow markdown across Claude/Codex/Antigravity. |
@@ -592,6 +599,7 @@ This warning comes from a transitive dependency in the local embedding/WASM stac
592
599
  | `ctx benchmark -- "task"` | Compares baseline AGENTS.md ordering with ContextOS task-aware scheduling. | You want a before/after signal for lost-in-the-middle risk. | Prints tables for parsed/actionable/filtered rules, baseline middle-risk, scheduled high/mid rules, recency reminder status, and top scored rules. |
593
600
  | `ctx benchmark --skills` | Runs the Skill Router eval benchmark. | You want evidence for skill routing accuracy and negative gates. | Prints top-1 accuracy, top-3 recall, false positive rate, confidence calibration, and negative gate accuracy across `eval/skill-routing` fixtures. |
594
601
  | `ctx leaderboard --hallucination` | Compares raw prompt-only skill guesses with ContextOS evidence routing. | You want launch evidence for the hallucination problem. | Runs 20 fixture tasks across 10+ repo contexts and prints Raw Agent vs ContextOS correctness plus sample failures. |
602
+ | `ctx leaderboard --agents codex,gemini` | Runs the same benchmark shape through installed agent CLIs. | You want real agent output instead of the deterministic raw baseline. | Calls `codex exec` in read-only mode and the local Gemini CLI with timeouts; missing or unauthenticated CLIs are reported as skipped/errors instead of blocking. |
595
603
  | `ctx sync --rules` | Syncs project rules and MCP servers through Ruler. | You want Codex, Claude Code, and Antigravity to share one project rule/MCP source of truth. | Ensures `.ruler/ruler.toml`, injects `ctx-mcp`, imports existing MCP servers from Codex and project `.mcp.json`, runs `ruler apply --agents codex,claude,antigravity`, mirrors MCP servers to Antigravity MCP configs, and verifies generated config. |
596
604
  | `ctx sync --rules --agents <list>` | Syncs only selected agents through Ruler. | You want to update one or two agents without touching the others. | Accepts comma-separated values such as `codex`, `claude`, `agy`, `antigravity`, or `codex,claude,agy`; `agy` is normalized to Ruler's `antigravity`. |
597
605
  | `ctx sync --rules --dry-run` | Previews Ruler sync without writing files or running apply. | You want to inspect behavior before changing project config. | Prints the same flow with dry-run status. |
package/bin/ctx.js CHANGED
@@ -21,6 +21,7 @@ import { installMcpTelemetryProxies } from "../plugins/ctx/lib/mcp-proxy-install
21
21
  import { benchmarkWorkspace, formatBenchmark } from "../plugins/ctx/lib/benchmark.js";
22
22
  import { formatSkillRoutingBenchmark, runSkillRoutingEval } from "../eval/skill-routing/run-eval.js";
23
23
  import { formatHallucinationLeaderboard, runHallucinationLeaderboard } from "../eval/hallucination/run-leaderboard.js";
24
+ import { formatAgentLeaderboard, runAgentLeaderboard } from "../eval/hallucination/run-agent-leaderboard.js";
24
25
  import { copyDir, copyPackageRoot, syncPackageRoot } from "../plugins/ctx/lib/package-install.js";
25
26
  import { installClaudeHooks } from "../plugins/ctx/lib/claude-hooks.js";
26
27
  import { installClaudeMcp } from "../plugins/ctx/lib/claude-mcp.js";
@@ -199,6 +200,7 @@ Usage:
199
200
  ctx benchmark -- "task" Benchmark workspace for a task
200
201
  ctx benchmark --skills Run skill routing eval benchmark
201
202
  ctx leaderboard --hallucination Compare raw agent guesses vs ContextOS routing
203
+ ctx leaderboard --agents codex,gemini Run live CLI leaderboard for installed agents
202
204
  ctx sync --rules Sync AGENTS.md rules to all agents
203
205
  ctx sync --rules --agents <names> Sync rules to specific agents only
204
206
  ctx sync --rules --dry-run Preview rule sync without writing
@@ -1039,8 +1041,19 @@ try {
1039
1041
  } else if (command === "leaderboard") {
1040
1042
  if (args.includes("--hallucination")) {
1041
1043
  console.log(formatHallucinationLeaderboard(await runHallucinationLeaderboard({ rootDir })));
1044
+ } else if (args.includes("--agents")) {
1045
+ const index = args.indexOf("--agents");
1046
+ const agents = String(args[index + 1] || "").split(",").map((agent) => agent.trim()).filter(Boolean);
1047
+ const limitIndex = args.indexOf("--limit");
1048
+ const timeoutIndex = args.indexOf("--timeout-ms");
1049
+ console.log(formatAgentLeaderboard(runAgentLeaderboard({
1050
+ rootDir,
1051
+ agents: agents.length ? agents : undefined,
1052
+ caseLimit: limitIndex >= 0 ? Number(args[limitIndex + 1]) : undefined,
1053
+ timeoutMs: timeoutIndex >= 0 ? Number(args[timeoutIndex + 1]) : undefined
1054
+ })));
1042
1055
  } else {
1043
- throw new Error("Usage: ctx leaderboard --hallucination");
1056
+ throw new Error("Usage: ctx leaderboard --hallucination OR ctx leaderboard --agents codex,gemini");
1044
1057
  }
1045
1058
  } else if (command === "skills") {
1046
1059
  if (args[1] === "doctor") {
Binary file
@@ -1,28 +1,25 @@
1
- $ cat AGENTS.md
2
- 1. General style
3
- 2. Formatting
4
- 3. Test names
5
- ...
6
- 37. IMPORTANT: Always use code-review-graph before grep.
7
- ...
8
- 52. Release notes
1
+ $ ctx benchmark -- "fix failing test"
2
+ ContextOS benchmark
9
3
 
10
- $ codex "fix failing test"
11
- Raw agent starts with grep
12
- Rule followed: no
4
+ Summary
5
+ -------
6
+ Metric Value
7
+ -------------------- -------------------------
8
+ Task fix failing test
9
+ Rules parsed 7
10
+ Actionable rules 5
11
+ Filtered rules 2
12
+ Relevant rules 1
13
+ Baseline middle-risk 1/1 relevant rules (100%)
14
+ ContextOS scheduled 1 high, 0 mid
15
+ Recency reminder enabled
13
16
 
14
- $ ctx debug -- "fix failing test"
15
- ContextOS debug
16
- Critical ContextOS rules:
17
- - IMPORTANT: Always use code-review-graph before grep.
18
-
19
- Suggested files to check:
20
- - test/score-context.test.js
21
- - plugins/ctx/lib/score-context.js
22
-
23
- $ codex + ContextOS
24
- Rule followed: yes
25
- Evidence: graph checked before file reads
26
-
27
- AGENTS.md did not change.
28
- The rule moved from buried context into runtime context.
17
+ Top Rules
18
+ ---------
19
+ Score Rule Reasons
20
+ ----- ---------------------------------------------------------------------------------------- ----------
21
+ 0.50 IMPORTANT: This project has a knowledge graph. ALWAYS use code-review-graph MCP tools... imperative
22
+ 0.00 AGENTS.md
23
+ 0.00 Centralised AI agent instructions. Add coding guidelines, style guides, and project c...
24
+ 0.00 Ruler concatenates all .md files in this directory (and subdirectories), starting wit...
25
+ 0.00 Use `query_graph` pattern="tests_for" to check coverage.
@@ -0,0 +1,76 @@
1
+ #!/usr/bin/env node
2
+ import { execFileSync } from "node:child_process";
3
+ import fs from "node:fs";
4
+ import path from "node:path";
5
+ import { fileURLToPath } from "node:url";
6
+
7
+ import { benchmarkWorkspace, formatBenchmark } from "../../plugins/ctx/lib/benchmark.js";
8
+ import { formatContextOSReady, inspectContextOSReady } from "../../plugins/ctx/lib/certification.js";
9
+ import { formatHallucinationLeaderboard, runHallucinationLeaderboard } from "../../eval/hallucination/run-leaderboard.js";
10
+ import { runSkillRoutingEval } from "../../eval/skill-routing/run-eval.js";
11
+
12
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
13
+ const repoRoot = path.resolve(__dirname, "..", "..");
14
+ const render = path.join(__dirname, "render-terminal-gif.mjs");
15
+ const leaderboard = await runHallucinationLeaderboard({ rootDir: repoRoot });
16
+ const skillEval = await runSkillRoutingEval({ rootDir: repoRoot });
17
+
18
+ const demos = [
19
+ {
20
+ log: "same-prompt-different-context.txt",
21
+ gif: "same-prompt-different-context.gif",
22
+ steps: [
23
+ ["ctx leaderboard --hallucination", formatHallucinationLeaderboard(leaderboard)],
24
+ ["ctx skills doctor -- \"fix deployed\" # Expo fixture", routeSummary("expo-eas", "fix deployed")],
25
+ ["ctx skills doctor -- \"fix deployed\" # Next/Vercel fixture", routeSummary("next-vercel", "fix deployed")]
26
+ ]
27
+ },
28
+ {
29
+ log: "agents-lost-middle.txt",
30
+ gif: "agents-lost-middle.gif",
31
+ steps: [
32
+ ["ctx benchmark -- \"fix failing test\"", formatBenchmark(benchmarkWorkspace({ cwd: repoRoot, task: "fix failing test" }))]
33
+ ]
34
+ },
35
+ {
36
+ log: "contextos-ready.txt",
37
+ gif: "contextos-ready.gif",
38
+ steps: [
39
+ ["ctx doctor", formatContextOSReady(inspectContextOSReady({ cwd: repoRoot }))]
40
+ ]
41
+ }
42
+ ];
43
+
44
+ for (const demo of demos) {
45
+ const logPath = path.join(__dirname, demo.log);
46
+ const gifPath = path.join(__dirname, demo.gif);
47
+ const chunks = [];
48
+ for (const [label, output] of demo.steps) {
49
+ chunks.push(`$ ${label}`);
50
+ chunks.push(cleanOutput(output));
51
+ chunks.push("");
52
+ }
53
+ fs.writeFileSync(logPath, chunks.join("\n").trimEnd() + "\n", "utf8");
54
+ execFileSync(process.execPath, [render, logPath, gifPath], { cwd: repoRoot, stdio: "inherit" });
55
+ }
56
+
57
+ function cleanOutput(output) {
58
+ return String(output || "")
59
+ .split(/\r?\n/)
60
+ .map((line) => line.trimEnd())
61
+ .join("\n")
62
+ .trim();
63
+ }
64
+
65
+ function routeSummary(fixture, prompt) {
66
+ const row = skillEval.rows.find((item) => item.fixture === fixture && item.prompt === prompt);
67
+ if (!row) return "No route found.";
68
+ return [
69
+ "ContextOS skill doctor",
70
+ `fixture: ${fixture}`,
71
+ `prompt: ${prompt}`,
72
+ `selected: ${row.selectedIds.join(", ") || "(none)"}`,
73
+ `expected: ${row.expected.join(", ") || "(none)"}`,
74
+ `rejected: ${row.forbidden.join(", ") || "(none)"}`
75
+ ].join("\n");
76
+ }
Binary file
@@ -12,9 +12,3 @@ Evidence:
12
12
  - Rules: 1 AGENTS.md source(s), 5 actionable rule(s)
13
13
  - Skills: 3 skill(s), 3 metadata file(s)
14
14
  - Workflows: 2 workflow(s), 2 with agent chain(s)
15
-
16
- $ badge
17
- [ContextOS Ready Gold]
18
-
19
- Repos now have a target:
20
- AGENTS.md + skills + workflows + evidence.
@@ -35,7 +35,7 @@ for (let count = 1; count <= displayLines.length; count += frameStep) {
35
35
  frames.push(writeFrame({ tmpDir, index: frames.length, lines: displayLines }));
36
36
 
37
37
  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
38
- execFileSync("convert", ["-delay", "12", "-loop", "0", ...frames, outputPath], { stdio: "inherit" });
38
+ execFileSync("convert", ["-limit", "time", "120", "-delay", "12", "-loop", "0", ...frames, outputPath], { stdio: "inherit" });
39
39
  console.log(`Wrote ${outputPath}`);
40
40
 
41
41
  function writeFrame({ tmpDir, index, lines }) {
@@ -8,19 +8,44 @@ System Correct Skill
8
8
  Raw Agent 10.0%
9
9
  ContextOS + Codex 80.0%
10
10
 
11
- $ ctx skills doctor -- "fix deployed" # Expo repo
12
- ContextOS skill doctor
13
- 1. eas high confidence
14
- evidence: eas.json, app.json, expo dependency
15
- 2. mobile-deployment high confidence
16
- 3. github-actions-ci-cd medium confidence
11
+ Sample failures:
12
+ - expo-eas: "fix deployed"
13
+ expected: eas, mobile-deployment, github-actions-ci-cd
14
+ raw: eas, env-secret-management, railway-render-deployment
15
+ contextos: eas, github-actions-ci-cd, mobile-deployment
16
+ - next-vercel: "fix deployed"
17
+ expected: vercel-deployment, github-actions-ci-cd, env-secret-management
18
+ raw: eas, env-secret-management, railway-render-deployment ✗
19
+ contextos: vercel-deployment, github-actions-ci-cd, env-secret-management ✓
20
+ - docker-node: "docker image build failed"
21
+ expected: docker, build-log-debugging
22
+ raw: docker, build-log-debugging, github-actions-ci-cd ✗
23
+ contextos: build-log-debugging, docker ✓
24
+ - railway-render: "Railway deploy health check failed"
25
+ expected: railway-render-deployment, build-log-debugging
26
+ raw: railway-render-deployment, build-log-debugging, firebase-hosting ✗
27
+ contextos: build-log-debugging, railway-render-deployment ✓
28
+ - firebase-hosting: "deploy firebase hosting"
29
+ expected: firebase-hosting
30
+ raw: firebase-hosting, flutter-firebase, railway-render-deployment ✗
31
+ contextos: firebase-hosting ✓
32
+ - nest-prisma: "optimize slow prisma queries"
33
+ expected: prisma, nestjs-module
34
+ raw: prisma, nestjs-module, android-signing ✗
35
+ contextos: nestjs-module, prisma ✓
17
36
 
18
- $ ctx skills doctor -- "fix deployed" # Next.js repo
37
+ $ ctx skills doctor -- "fix deployed" # Expo fixture
19
38
  ContextOS skill doctor
20
- 1. vercel-deployment high confidence
21
- evidence: vercel.json, next dependency
22
- 2. github-actions-ci-cd high confidence
23
- 3. env-secret-management medium confidence
39
+ fixture: expo-eas
40
+ prompt: fix deployed
41
+ selected: eas, github-actions-ci-cd, mobile-deployment
42
+ expected: eas, mobile-deployment, github-actions-ci-cd
43
+ rejected: vercel-deployment
24
44
 
25
- Same prompt. Same model. Different repo evidence.
26
- ContextOS routes the right skill before the agent edits code.
45
+ $ ctx skills doctor -- "fix deployed" # Next/Vercel fixture
46
+ ContextOS skill doctor
47
+ fixture: next-vercel
48
+ prompt: fix deployed
49
+ selected: vercel-deployment, github-actions-ci-cd, env-secret-management
50
+ expected: vercel-deployment, github-actions-ci-cd, env-secret-management
51
+ rejected: eas
@@ -0,0 +1,233 @@
1
+ #!/usr/bin/env node
2
+ import { execFileSync, spawnSync } from "node:child_process";
3
+ import fs from "node:fs";
4
+ import os from "node:os";
5
+ import path from "node:path";
6
+ import { fileURLToPath } from "node:url";
7
+
8
+ import { parseEvalYaml } from "../skill-routing/run-eval.js";
9
+
10
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
11
+ const repoRoot = path.resolve(__dirname, "..", "..");
12
+ const skillEvalRoot = path.resolve(__dirname, "..", "skill-routing");
13
+ const DEFAULT_AGENTS = ["codex", "gemini"];
14
+ const DEFAULT_CASE_LIMIT = 5;
15
+ const DEFAULT_TIMEOUT_MS = 120000;
16
+
17
+ export function runAgentLeaderboard({
18
+ agents = DEFAULT_AGENTS,
19
+ casesPath = path.join(skillEvalRoot, "cases.yaml"),
20
+ caseLimit = DEFAULT_CASE_LIMIT,
21
+ timeoutMs = DEFAULT_TIMEOUT_MS,
22
+ rootDir = repoRoot
23
+ } = {}) {
24
+ const config = parseEvalYaml(fs.readFileSync(casesPath, "utf8"));
25
+ const cases = config.cases
26
+ .filter((row) => row.expected?.length)
27
+ .slice(0, caseLimit);
28
+ const skillIds = config.skills.map((skill) => skill.id);
29
+ const systems = [];
30
+
31
+ for (const agent of agents) {
32
+ const binary = findBinary(agent);
33
+ if (!binary) {
34
+ systems.push({ name: agent, status: "skipped", reason: "binary not found", rows: [], correctRate: 0 });
35
+ continue;
36
+ }
37
+ const rows = cases.map((testCase) => runAgentCase({
38
+ agent,
39
+ binary,
40
+ testCase,
41
+ skillIds,
42
+ timeoutMs,
43
+ rootDir
44
+ }));
45
+ const completed = rows.filter((row) => row.status === "ok");
46
+ const correct = completed.filter((row) => row.correct).length;
47
+ systems.push({
48
+ name: agent,
49
+ status: completed.length ? "ok" : "skipped",
50
+ reason: completed.length ? "" : firstReason(rows),
51
+ rows,
52
+ correctRate: completed.length ? correct / completed.length : 0
53
+ });
54
+ }
55
+
56
+ return {
57
+ mode: "live-agent-cli",
58
+ caseCount: cases.length,
59
+ systems
60
+ };
61
+ }
62
+
63
+ export function formatAgentLeaderboard(result) {
64
+ const lines = [
65
+ "Live Agent Leaderboard",
66
+ `Mode: ${result.mode}`,
67
+ `Tasks: ${result.caseCount}`,
68
+ "",
69
+ "System Status Correct Skill",
70
+ "-------- -------- -------------"
71
+ ];
72
+ for (const system of result.systems) {
73
+ const score = system.status === "ok" ? percent(system.correctRate) : system.reason;
74
+ lines.push(`${system.name.padEnd(8)} ${system.status.padEnd(8)} ${score}`);
75
+ }
76
+ lines.push("", "Cases:");
77
+ for (const system of result.systems) {
78
+ lines.push(`- ${system.name}`);
79
+ for (const row of system.rows.slice(0, 5)) {
80
+ lines.push(` - ${row.fixture}: "${row.prompt}"`);
81
+ lines.push(` selected: ${row.selectedIds.join(", ") || "(none)"} ${row.correct ? "✓" : "✗"}`);
82
+ if (row.status !== "ok") lines.push(` status: ${row.status}; ${row.reason}`);
83
+ }
84
+ }
85
+ return lines.join("\n");
86
+ }
87
+
88
+ function runAgentCase({ agent, binary, testCase, skillIds, timeoutMs, rootDir }) {
89
+ const cwd = testCase.fixture === "contextos"
90
+ ? rootDir
91
+ : path.join(skillEvalRoot, "fixtures", testCase.fixture);
92
+ const prompt = buildPrompt({ task: testCase.prompt, skillIds });
93
+ const startedAt = Date.now();
94
+ const result = spawnSync(binary, agentArgs({ agent, cwd, prompt }), {
95
+ cwd,
96
+ encoding: "utf8",
97
+ timeout: timeoutMs,
98
+ maxBuffer: 1024 * 1024 * 4,
99
+ env: {
100
+ ...process.env,
101
+ NO_COLOR: "1",
102
+ CI: process.env.CI || "1"
103
+ }
104
+ });
105
+ const output = `${result.stdout || ""}\n${result.stderr || ""}`.trim();
106
+ const selectedIds = parseSkillIds(output, skillIds).slice(0, 3);
107
+ const correct = isCorrect({
108
+ selectedIds,
109
+ expected: testCase.expected || [],
110
+ allowed: testCase.allowed || [],
111
+ forbidden: testCase.forbidden || []
112
+ });
113
+ return {
114
+ status: result.error ? "error" : result.status === 0 ? "ok" : "error",
115
+ reason: result.error?.message || (result.status === 0 ? "" : `exit ${result.status}`),
116
+ prompt: testCase.prompt,
117
+ fixture: testCase.fixture,
118
+ expected: testCase.expected || [],
119
+ selectedIds,
120
+ correct,
121
+ durationMs: Date.now() - startedAt
122
+ };
123
+ }
124
+
125
+ function agentArgs({ agent, cwd, prompt }) {
126
+ if (agent === "codex") {
127
+ return [
128
+ "exec",
129
+ "--cd", cwd,
130
+ "--sandbox", "read-only",
131
+ "--ask-for-approval", "never",
132
+ prompt
133
+ ];
134
+ }
135
+ if (agent === "gemini") {
136
+ const template = process.env.CONTEXTOS_GEMINI_CMD;
137
+ if (template) return expandTemplate(template, { cwd, prompt });
138
+ return ["-p", prompt];
139
+ }
140
+ return [prompt];
141
+ }
142
+
143
+ function buildPrompt({ task, skillIds }) {
144
+ return [
145
+ "You are evaluating a repository for a coding-agent skill router benchmark.",
146
+ "Do not edit files. Do not run commands.",
147
+ `Task: ${task}`,
148
+ `Allowed skill IDs: ${skillIds.join(", ")}`,
149
+ "Return only the top skill IDs as comma-separated text. No explanations."
150
+ ].join("\n");
151
+ }
152
+
153
+ function parseSkillIds(output, skillIds) {
154
+ const normalized = new Map(skillIds.map((id) => [normalize(id), id]));
155
+ const found = [];
156
+ for (const token of String(output || "").split(/[^A-Za-z0-9_.@-]+/)) {
157
+ const id = normalized.get(normalize(token));
158
+ if (id && !found.includes(id)) found.push(id);
159
+ }
160
+ return found;
161
+ }
162
+
163
+ function isCorrect({ selectedIds, expected, allowed, forbidden }) {
164
+ const selected = new Set(selectedIds);
165
+ const accepted = new Set([...expected, ...allowed]);
166
+ return expected.every((skill) => selected.has(skill))
167
+ && forbidden.every((skill) => !selected.has(skill))
168
+ && selectedIds.every((skill) => accepted.has(skill));
169
+ }
170
+
171
+ function findBinary(name) {
172
+ const safeName = shellQuote(name);
173
+ const candidates = [
174
+ path.join(os.homedir(), ".local", "bin", safeName),
175
+ path.join(os.homedir(), ".npm-global", "bin", safeName),
176
+ path.join(os.homedir(), ".nvm", "current", "bin", safeName),
177
+ `/mnt/c/Users/admin/AppData/Roaming/npm/${safeName}`,
178
+ `/mnt/c/Users/admin/AppData/Roaming/npm/${safeName}.cmd`
179
+ ];
180
+ for (const candidate of candidates) {
181
+ if (fs.existsSync(candidate)) return candidate;
182
+ }
183
+ for (const command of [
184
+ `command -v ${safeName}`,
185
+ `source ~/.profile >/dev/null 2>&1 || true; source ~/.bashrc >/dev/null 2>&1 || true; command -v ${safeName}`
186
+ ]) {
187
+ try {
188
+ const found = execFileSync("bash", ["-lc", command], { encoding: "utf8" }).trim();
189
+ if (found) return found;
190
+ } catch {
191
+ // continue
192
+ }
193
+ }
194
+ return "";
195
+ }
196
+
197
+ function firstReason(rows) {
198
+ return rows.find((row) => row.reason)?.reason || "no completed cases";
199
+ }
200
+
201
+ function normalize(value) {
202
+ return String(value || "").toLowerCase().replace(/[^a-z0-9]+/g, "");
203
+ }
204
+
205
+ function shellQuote(value) {
206
+ return String(value).replace(/[^A-Za-z0-9_./-]/g, "");
207
+ }
208
+
209
+ function expandTemplate(template, vars) {
210
+ const file = path.join(os.tmpdir(), `contextos-agent-prompt-${process.pid}-${Date.now()}.txt`);
211
+ fs.writeFileSync(file, vars.prompt, "utf8");
212
+ return String(template)
213
+ .replaceAll("{cwd}", vars.cwd)
214
+ .replaceAll("{prompt_file}", file)
215
+ .split(/\s+/)
216
+ .filter(Boolean);
217
+ }
218
+
219
+ function percent(value) {
220
+ return `${(value * 100).toFixed(1)}%`;
221
+ }
222
+
223
+ if (import.meta.url === `file://${process.argv[1]}`) {
224
+ const agentsArg = process.argv.find((arg) => arg.startsWith("--agents="));
225
+ const limitArg = process.argv.find((arg) => arg.startsWith("--limit="));
226
+ const timeoutArg = process.argv.find((arg) => arg.startsWith("--timeout-ms="));
227
+ const result = runAgentLeaderboard({
228
+ agents: agentsArg ? agentsArg.slice("--agents=".length).split(",").map((item) => item.trim()).filter(Boolean) : DEFAULT_AGENTS,
229
+ caseLimit: limitArg ? Number(limitArg.slice("--limit=".length)) : DEFAULT_CASE_LIMIT,
230
+ timeoutMs: timeoutArg ? Number(timeoutArg.slice("--timeout-ms=".length)) : DEFAULT_TIMEOUT_MS
231
+ });
232
+ console.log(formatAgentLeaderboard(result));
233
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@minhpnq1807/contextos",
3
- "version": "0.6.1",
3
+ "version": "0.6.2",
4
4
  "description": "Task-aware AGENTS.md context injection and compliance reporting for Codex, Claude Code, and Antigravity.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -10,6 +10,7 @@
10
10
  "files": [
11
11
  "bin/",
12
12
  "plugins/",
13
+ "scripts/",
13
14
  ".codex/skills/",
14
15
  ".codex/workflows/",
15
16
  ".agents/",
@@ -28,6 +29,9 @@
28
29
  "validate:plugin": "node test/validate-plugin.js",
29
30
  "benchmark:skills": "node bin/ctx.js benchmark --skills",
30
31
  "leaderboard:hallucination": "node eval/hallucination/run-leaderboard.js",
32
+ "leaderboard:agents": "node eval/hallucination/run-agent-leaderboard.js --agents=codex,gemini",
33
+ "demo:capture": "node docs/demo/capture-live-demos.mjs",
34
+ "sync:community-skills": "node scripts/sync-community-skills.mjs",
31
35
  "test:mcp": "node test/mcp-protocol-smoke.js"
32
36
  },
33
37
  "engines": {
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ctx",
3
- "version": "0.6.1",
3
+ "version": "0.6.2",
4
4
  "description": "Inject task-relevant AGENTS.md rules into Codex through plugin hooks.",
5
5
  "author": {
6
6
  "name": "ContextOS"
@@ -0,0 +1,40 @@
1
+ #!/usr/bin/env node
2
+ import fs from "node:fs";
3
+ import path from "node:path";
4
+
5
+ const [, , sourceArg = "external-skills", targetArg = "community-skills"] = process.argv;
6
+ const source = path.resolve(sourceArg);
7
+ const target = path.resolve(targetArg);
8
+ const skip = new Set([".git", ".github", "scripts"]);
9
+
10
+ if (!fs.existsSync(source)) {
11
+ console.error(`Missing source skills directory: ${source}`);
12
+ process.exit(1);
13
+ }
14
+
15
+ fs.mkdirSync(target, { recursive: true });
16
+
17
+ for (const entry of fs.readdirSync(target, { withFileTypes: true })) {
18
+ if (entry.isDirectory()) fs.rmSync(path.join(target, entry.name), { recursive: true, force: true });
19
+ else if (entry.isFile() && entry.name !== "README.md") fs.rmSync(path.join(target, entry.name), { force: true });
20
+ }
21
+
22
+ for (const entry of fs.readdirSync(source, { withFileTypes: true })) {
23
+ if (skip.has(entry.name)) continue;
24
+ const from = path.join(source, entry.name);
25
+ const to = path.join(target, entry.name);
26
+ if (entry.isDirectory()) copyDir(from, to);
27
+ else if (entry.isFile() && entry.name === "README.md") fs.copyFileSync(from, to);
28
+ }
29
+
30
+ console.log(`Synced ContextOS skills from ${source} to ${target}`);
31
+
32
+ function copyDir(from, to) {
33
+ fs.mkdirSync(to, { recursive: true });
34
+ for (const entry of fs.readdirSync(from, { withFileTypes: true })) {
35
+ const sourcePath = path.join(from, entry.name);
36
+ const targetPath = path.join(to, entry.name);
37
+ if (entry.isDirectory()) copyDir(sourcePath, targetPath);
38
+ else if (entry.isFile()) fs.copyFileSync(sourcePath, targetPath);
39
+ }
40
+ }