nccgs 2.1.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/nccgs-adversarial-reviewer.md +1 -1
- package/.claude/agents/nccgs-engine-programmer.md +1 -1
- package/.claude/agents/nccgs-fast-implementer.md +1 -1
- package/.claude/agents/nccgs-game-designer.md +1 -1
- package/.claude/agents/nccgs-gameplay-programmer.md +1 -1
- package/.claude/agents/nccgs-qa-engineer.md +1 -1
- package/.claude/agents/nccgs-task-scout.md +1 -1
- package/.claude/agents/nccgs-ui-programmer.md +1 -1
- package/.claude/agents/nccgs-unity-build-specialist.md +1 -1
- package/.claude/agents/nccgs-unity-implementer.md +1 -1
- package/.claude/agents/nccgs-unity-systems-specialist.md +1 -1
- package/.claude/agents/nccgs-unity-ui-specialist.md +1 -1
- package/.claude/agents/nccgs-verification-engineer.md +1 -1
- package/.claude/nccgs/MEMORY.md +3 -1
- package/.claude/nccgs/VERSION +1 -1
- package/.claude/nccgs/constitution.md +187 -176
- package/.claude/nccgs/hooks/closure-guard.mjs +2 -0
- package/.claude/nccgs/hooks/unity-transport-guard.mjs +11 -0
- package/.claude/nccgs/pipeline.json +260 -260
- package/.claude/nccgs/protocols/context-packets.md +5 -0
- package/.claude/nccgs/protocols/model-routing.md +46 -46
- package/.claude/nccgs/protocols/review-handoff.md +21 -1
- package/.claude/nccgs/protocols/shared-workflow-context.md +33 -0
- package/.claude/nccgs/protocols/unity-boundary.md +20 -0
- package/.claude/nccgs/routing.generated.md +4 -4
- package/.claude/nccgs/settings.fragment.json +84 -24
- package/.claude/nccgs/studio.json +53 -53
- package/.claude/nccgs/tools/autonomy.mjs +13 -0
- package/.claude/nccgs/tools/command-tokens.mjs +22 -0
- package/.claude/nccgs/tools/hook-transport.mjs +34 -0
- package/.claude/nccgs/tools/native-completion.mjs +10 -1
- package/.claude/nccgs/tools/returned-handoff.mjs +11 -2
- package/.claude/nccgs/tools/shared-context-guard.mjs +60 -0
- package/.claude/nccgs/tools/shared-context.mjs +25 -0
- package/.claude/nccgs/tools/tool-guard.mjs +120 -131
- package/.claude/nccgs/tools/unity-transport.mjs +170 -0
- package/.claude/nccgs/workflow-catalog.json +4 -4
- package/CHANGELOG.md +235 -200
- package/CLAUDE.md +10 -8
- package/MEMORY.md +109 -60
- package/README.md +206 -191
- package/UPGRADING.md +130 -118
- package/VERSION +1 -1
- package/docs/ARCHITECTURE.md +131 -131
- package/docs/ASSET-PRODUCTION-GRAPHIFY-PLAN.md +230 -0
- package/docs/HUONG-DAN-MIGRATE-VA-SU-DUNG.md +117 -116
- package/docs/NCCGS-1.5.md +237 -237
- package/docs/PROJECT-POLICY.md +94 -94
- package/docs/RELEASE-STATUS.md +17 -9
- package/docs/UNINSTALL.md +3 -3
- package/docs/WFK17-HOOK-REPAIR.md +71 -0
- package/docs/WORKFLOWS.md +85 -85
- package/package.json +1 -1
- package/scaffold/.nccgs/project.yaml +109 -108
- package/scripts/benchmark-v15.mjs +173 -173
- package/scripts/doctor.mjs +261 -261
- package/scripts/install.mjs +530 -529
- package/scripts/run.mjs +5 -4
- package/scripts/supervise.mjs +4 -0
- package/scripts/uninstall.mjs +292 -292
- package/scripts/validate.mjs +285 -280
- package/tests/autonomy.test.mjs +126 -113
- package/tests/design-readiness.test.mjs +199 -199
- package/tests/fixtures/compiler-shutdown.cs +67 -64
- package/tests/fixtures/native-provider.mjs +61 -0
- package/tests/framework.test.mjs +740 -740
- package/tests/hook-repair.test.mjs +106 -0
- package/tests/lifecycle-v15.test.mjs +346 -300
- package/tests/native-completion-notices.test.mjs +181 -0
- package/tests/phase1.test.mjs +427 -416
- package/tests/phase2.test.mjs +250 -250
- package/tests/policy-v15.test.mjs +234 -234
- package/tests/returned-handoff-notices.test.mjs +170 -0
- package/tests/runtime-v15.test.mjs +1197 -1191
- package/tests/task-slices.test.mjs +285 -285
- package/tests/unity-transport.test.mjs +348 -0
|
@@ -1,108 +1,109 @@
|
|
|
1
|
-
schema_version: 2
|
|
2
|
-
|
|
3
|
-
project:
|
|
4
|
-
name: "[set during migration/onboarding]"
|
|
5
|
-
stage: existing
|
|
6
|
-
platforms: []
|
|
7
|
-
|
|
8
|
-
studio:
|
|
9
|
-
profile: unity-production
|
|
10
|
-
enabled_departments: [leadership, design, programming, unity, content, quality, release]
|
|
11
|
-
orchestration:
|
|
12
|
-
max_agent_depth: 1
|
|
13
|
-
max_parallel_agents: 2
|
|
14
|
-
single_writer_per_path: true
|
|
15
|
-
independent_verification: true
|
|
16
|
-
|
|
17
|
-
models:
|
|
18
|
-
profile: claude-standalone
|
|
19
|
-
allow_inherit_fallback: false
|
|
20
|
-
allow_automatic_escalation: false
|
|
21
|
-
notify_on_escalation: true
|
|
22
|
-
overrides: {}
|
|
23
|
-
|
|
24
|
-
# Profile contracts are resolved from this canonical configuration.
|
|
25
|
-
pipeline_policy: .claude/nccgs/pipeline.json
|
|
26
|
-
|
|
27
|
-
workflow:
|
|
28
|
-
operating_mode: solo
|
|
29
|
-
autonomy: high
|
|
30
|
-
default_classification: auto
|
|
31
|
-
documentation: targeted
|
|
32
|
-
traceability: targeted
|
|
33
|
-
review_intensity: risk_based
|
|
34
|
-
|
|
35
|
-
context:
|
|
36
|
-
packet_path: ".nccgs/context/{feature_id}.yaml"
|
|
37
|
-
full_source_reads: on_demand
|
|
38
|
-
# Advisory context targets, not measured token/spend limits. Review-round limits
|
|
39
|
-
# are executable and resolved from pipeline.json contract.taskTiers.
|
|
40
|
-
budgets: { fast: 4000, standard: 12000, controlled: 24000 }
|
|
41
|
-
|
|
42
|
-
authority:
|
|
43
|
-
implementation_details: claude
|
|
44
|
-
safe_refactor: claude
|
|
45
|
-
automated_verification: claude
|
|
46
|
-
technical_docs_sync: claude
|
|
47
|
-
gameplay_change: product_owner
|
|
48
|
-
scope_change: product_owner
|
|
49
|
-
subjective_acceptance: product_owner
|
|
50
|
-
dependency_or_service: product_owner
|
|
51
|
-
irreversible_migration: product_owner
|
|
52
|
-
|
|
53
|
-
canonical_sources:
|
|
54
|
-
product_decisions: []
|
|
55
|
-
game_design: []
|
|
56
|
-
architecture: []
|
|
57
|
-
planning: []
|
|
58
|
-
art_direction: []
|
|
59
|
-
narrative: []
|
|
60
|
-
audio: []
|
|
61
|
-
release: []
|
|
62
|
-
|
|
63
|
-
unity:
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
1
|
+
schema_version: 2
|
|
2
|
+
|
|
3
|
+
project:
|
|
4
|
+
name: "[set during migration/onboarding]"
|
|
5
|
+
stage: existing
|
|
6
|
+
platforms: []
|
|
7
|
+
|
|
8
|
+
studio:
|
|
9
|
+
profile: unity-production
|
|
10
|
+
enabled_departments: [leadership, design, programming, unity, content, quality, release]
|
|
11
|
+
orchestration:
|
|
12
|
+
max_agent_depth: 1
|
|
13
|
+
max_parallel_agents: 2
|
|
14
|
+
single_writer_per_path: true
|
|
15
|
+
independent_verification: true
|
|
16
|
+
|
|
17
|
+
models:
|
|
18
|
+
profile: claude-standalone
|
|
19
|
+
allow_inherit_fallback: false
|
|
20
|
+
allow_automatic_escalation: false
|
|
21
|
+
notify_on_escalation: true
|
|
22
|
+
overrides: {}
|
|
23
|
+
|
|
24
|
+
# Profile contracts are resolved from this canonical configuration.
|
|
25
|
+
pipeline_policy: .claude/nccgs/pipeline.json
|
|
26
|
+
|
|
27
|
+
workflow:
|
|
28
|
+
operating_mode: solo
|
|
29
|
+
autonomy: high
|
|
30
|
+
default_classification: auto
|
|
31
|
+
documentation: targeted
|
|
32
|
+
traceability: targeted
|
|
33
|
+
review_intensity: risk_based
|
|
34
|
+
|
|
35
|
+
context:
|
|
36
|
+
packet_path: ".nccgs/context/{feature_id}.yaml"
|
|
37
|
+
full_source_reads: on_demand
|
|
38
|
+
# Advisory context targets, not measured token/spend limits. Review-round limits
|
|
39
|
+
# are executable and resolved from pipeline.json contract.taskTiers.
|
|
40
|
+
budgets: { fast: 4000, standard: 12000, controlled: 24000 }
|
|
41
|
+
|
|
42
|
+
authority:
|
|
43
|
+
implementation_details: claude
|
|
44
|
+
safe_refactor: claude
|
|
45
|
+
automated_verification: claude
|
|
46
|
+
technical_docs_sync: claude
|
|
47
|
+
gameplay_change: product_owner
|
|
48
|
+
scope_change: product_owner
|
|
49
|
+
subjective_acceptance: product_owner
|
|
50
|
+
dependency_or_service: product_owner
|
|
51
|
+
irreversible_migration: product_owner
|
|
52
|
+
|
|
53
|
+
canonical_sources:
|
|
54
|
+
product_decisions: []
|
|
55
|
+
game_design: []
|
|
56
|
+
architecture: []
|
|
57
|
+
planning: []
|
|
58
|
+
art_direction: []
|
|
59
|
+
narrative: []
|
|
60
|
+
audio: []
|
|
61
|
+
release: []
|
|
62
|
+
|
|
63
|
+
unity:
|
|
64
|
+
# Enforced package boundary: direct CLI/Pipeline only; no Unity MCP fallback.
|
|
65
|
+
version: detect
|
|
66
|
+
render_pipeline: detect
|
|
67
|
+
ui_system: detect
|
|
68
|
+
input_system: detect
|
|
69
|
+
cli: enabled
|
|
70
|
+
pipeline: enabled
|
|
71
|
+
connected_editor_preferred: true
|
|
72
|
+
|
|
73
|
+
architecture:
|
|
74
|
+
decisions: []
|
|
75
|
+
forbidden_patterns: []
|
|
76
|
+
approved_packages: []
|
|
77
|
+
approved_services: []
|
|
78
|
+
|
|
79
|
+
ownership:
|
|
80
|
+
protected_paths: []
|
|
81
|
+
user_owned_paths: []
|
|
82
|
+
user_owned_settings: []
|
|
83
|
+
generated_paths: []
|
|
84
|
+
|
|
85
|
+
verification:
|
|
86
|
+
editmode: detect
|
|
87
|
+
playmode: detect
|
|
88
|
+
runtime: unity_cli
|
|
89
|
+
build_targets: []
|
|
90
|
+
required_regressions: []
|
|
91
|
+
recurring_manual_gates: []
|
|
92
|
+
baseline_policy: no_unexplained_reduction
|
|
93
|
+
|
|
94
|
+
closure:
|
|
95
|
+
mode: lean
|
|
96
|
+
canonical_record: ".nccgs/closures/{feature_id}.md"
|
|
97
|
+
required_artifacts: []
|
|
98
|
+
done_requires: [implementation, objective_verification, required_artifacts, applicable_manual_gates]
|
|
99
|
+
|
|
100
|
+
release:
|
|
101
|
+
protected_branches: [main, master]
|
|
102
|
+
required_builds: []
|
|
103
|
+
publishing_requires_product_owner: true
|
|
104
|
+
|
|
105
|
+
conventions:
|
|
106
|
+
source_paths: []
|
|
107
|
+
test_paths: []
|
|
108
|
+
documentation_paths: []
|
|
109
|
+
notes: []
|
|
@@ -1,173 +1,173 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import fs from "node:fs";
|
|
3
|
-
import os from "node:os";
|
|
4
|
-
import path from "node:path";
|
|
5
|
-
import { performance } from "node:perf_hooks";
|
|
6
|
-
import { fileURLToPath } from "node:url";
|
|
7
|
-
import { atomicJson, hash, json } from "../.claude/nccgs/tools/policy.mjs";
|
|
8
|
-
import { initialize, record, snapshot, taskPath } from "../.claude/nccgs/tools/gates.mjs";
|
|
9
|
-
import { reserve, reconcile, status } from "../.claude/nccgs/tools/orchestration.mjs";
|
|
10
|
-
|
|
11
|
-
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
12
|
-
const label = "SYNTHETIC FRAMEWORK BENCHMARK — no provider/model or game execution";
|
|
13
|
-
export const scenarios = [
|
|
14
|
-
{ id: "tiny-fix", name: "Tiny local fix", mode: "implement", scope: "local", target: "fix", expectedRisk: "FAST", objective: "Fix a local display label alignment with a known expected value" },
|
|
15
|
-
{ id: "learning-prototype", name: "Learning prototype", mode: "implement", scope: "feature", target: "learning-prototype", expectedRisk: "STANDARD", objective: "Explore a bounded feedback hypothesis in a synthetic prototype" },
|
|
16
|
-
{ id: "alpha-ready-prototype", name: "Alpha-ready prototype", mode: "implement", scope: "feature", target: "alpha-ready", expectedRisk: "STANDARD", objective: "Prepare a prototype feature for a declared end-to-end readiness check" },
|
|
17
|
-
{ id: "cross-system-design", name: "Cross-system optimization and design plan", mode: "plan", scope: "cross-system", target: "production-change", expectedRisk: "CONTROLLED", objective: "Plan cross-system allocation and update-order optimization from a measured baseline" },
|
|
18
|
-
{ id: "large-system-audit", name: "Large feature and system audit", mode: "audit", scope: "cross-system", target: "production-change", expectedRisk: "CONTROLLED", objective: "Audit dependencies and acceptance coverage across a large feature's systems" }
|
|
19
|
-
];
|
|
20
|
-
const measure = action => {
|
|
21
|
-
const start = performance.now(), value = action();
|
|
22
|
-
return { value, ms: performance.now() - start };
|
|
23
|
-
};
|
|
24
|
-
const rounded = value => Math.round(value * 1000) / 1000;
|
|
25
|
-
function distribution(values) {
|
|
26
|
-
const sorted = [...values].sort((a, b) => a - b), mid = Math.floor(sorted.length / 2);
|
|
27
|
-
return { samples: sorted.length, min: rounded(sorted[0]), median: rounded(sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2), max: rounded(sorted.at(-1)), total: rounded(sorted.reduce((a, b) => a + b, 0)) };
|
|
28
|
-
}
|
|
29
|
-
function workspace(sourceFiles) {
|
|
30
|
-
const project = fs.mkdtempSync(path.join(os.tmpdir(), "nccgs-v15-benchmark-"));
|
|
31
|
-
function write(relative, value) {
|
|
32
|
-
const file = path.join(project, relative);
|
|
33
|
-
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
34
|
-
fs.writeFileSync(file, typeof value === "string" ? value : `${JSON.stringify(value, null, 2)}\n`);
|
|
35
|
-
}
|
|
36
|
-
for (const file of ["pipeline.json", "studio.json"]) write(`.claude/nccgs/${file}`, fs.readFileSync(path.join(root, ".claude/nccgs", file), "utf8"));
|
|
37
|
-
write(".nccgs/project.yaml", "schema_version: 2\nmodels:\n profile: claude-standalone\n");
|
|
38
|
-
write(".nccgs/execution.json", { schemaVersion: 1, name: "standalone", externalReview: { mode: "off" }, acceptanceRequired: false, requireObservedEffort: false, maxParallelWorkers: 2 });
|
|
39
|
-
for (let i = 0; i < sourceFiles; i++) write(`SyntheticSource/component-${i}.txt`, `Synthetic framework fixture component ${i}.\n${"Boundary and dependency data only.\n".repeat(24)}`);
|
|
40
|
-
return { project, write,
|
|
41
|
-
init(id, scenario, parentId = null) {
|
|
42
|
-
const relative = `.nccgs/intakes/${id}.json`;
|
|
43
|
-
write(relative, { schemaVersion: 1, objective: scenario.objective, mode: scenario.mode, scope: scenario.scope, target: scenario.target, readiness: "ready", systems: ["synthetic-component"], requiredChecks: ["fixture-acceptance"] });
|
|
44
|
-
return initialize(project, id, null, null, null, { intake: relative, parentId });
|
|
45
|
-
},
|
|
46
|
-
dispose() {
|
|
47
|
-
const resolved = path.resolve(project), expectedParent = path.resolve(os.tmpdir());
|
|
48
|
-
assert.equal(path.dirname(resolved), expectedParent);
|
|
49
|
-
assert.ok(path.basename(resolved).startsWith("nccgs-v15-benchmark-"));
|
|
50
|
-
fs.rmSync(resolved, { recursive: true, force: true });
|
|
51
|
-
} };
|
|
52
|
-
}
|
|
53
|
-
function request(task, key, maxDispatches = null) {
|
|
54
|
-
// The stress parameter counts ordinary delivery attempts; protected slots stay
|
|
55
|
-
// inside the actual total ceiling and must remain untouched by this fixture.
|
|
56
|
-
const protectedSlots = Object.values(task.contract.deliveryReserve ?? {}).reduce((a, b) => a + b, 0);
|
|
57
|
-
return { kind: task.mode === "implement" ? "implementation" : "analysis", agent: task.mode === "implement" ? "nccgs-gameplay-programmer" : "nccgs-task-scout",
|
|
58
|
-
requestKey: key, reason: "Synthetic reservation only; no worker/provider is invoked", writePaths: task.mode === "implement" ? ["SyntheticSource"] : [],
|
|
59
|
-
...(maxDispatches === null ? {} : { limits: { maxDispatches: maxDispatches + protectedSlots } }) };
|
|
60
|
-
}
|
|
61
|
-
function cancelFixture(project, dispatch) {
|
|
62
|
-
return reconcile(project, dispatch.id, { status: "cancelled", reason: "Synthetic framework reservation completed without dispatching a model", reference: "benchmark-v15: intentionally no provider invocation" });
|
|
63
|
-
}
|
|
64
|
-
function scenarioRun(scenario, iterations, sourceFiles) {
|
|
65
|
-
const times = { init: [], snapshot: [], reserve: [], report: [] }, counts = { initialized: 0, snapshots: 0, reservations: 0, reports: 0, providerAttempts: 0 };
|
|
66
|
-
let source = null, observed = null;
|
|
67
|
-
for (let i = 0; i < iterations; i++) {
|
|
68
|
-
const fixture = workspace(sourceFiles);
|
|
69
|
-
try {
|
|
70
|
-
const initialized = measure(() => fixture.init(scenario.id, scenario));
|
|
71
|
-
const task = initialized.value;
|
|
72
|
-
assert.equal(task.mode, scenario.mode); assert.equal(task.risk, scenario.expectedRisk);
|
|
73
|
-
assert.equal(task.schemaVersion, 2);
|
|
74
|
-
const expectedRoute = { FAST: { class: "lightweight", model: "sonnet", effort: "
|
|
75
|
-
assert.deepEqual(task.implementationRoute, expectedRoute);
|
|
76
|
-
observed = { mode: task.mode, risk: task.risk, implementationRoute: task.implementationRoute };
|
|
77
|
-
times.init.push(initialized.ms); counts.initialized++;
|
|
78
|
-
const captured = measure(() => snapshot(fixture.project, task));
|
|
79
|
-
times.snapshot.push(captured.ms); counts.snapshots++;
|
|
80
|
-
source = { syntheticFiles: sourceFiles, snapshotFiles: captured.value.files, snapshotBytes: captured.value.bytes };
|
|
81
|
-
const reserved = measure(() => reserve(fixture.project, task.id, request(task, "measurement")));
|
|
82
|
-
times.reserve.push(reserved.ms); counts.reservations++;
|
|
83
|
-
cancelFixture(fixture.project, reserved.value);
|
|
84
|
-
const reported = measure(() => record(fixture.project, task.id, "report", { outcome: "blocked", summary: `${label}. Classification, snapshot, reservation and reporting were exercised; there is no runtime/model evidence.` }));
|
|
85
|
-
times.report.push(reported.ms); counts.reports++;
|
|
86
|
-
const usage = status(fixture.project, task.id);
|
|
87
|
-
assert.equal(usage.usage.dispatches, 1);
|
|
88
|
-
assert.equal(usage.telemetry.attempts, 0);
|
|
89
|
-
assert.equal(reported.value.reviewBudget.used, 0);
|
|
90
|
-
assert.equal(reported.value.stages.report.verificationState.verified, false);
|
|
91
|
-
assert.equal(reported.value.outcome, "blocked");
|
|
92
|
-
} finally { fixture.dispose(); }
|
|
93
|
-
}
|
|
94
|
-
return { id: scenario.id, name: scenario.name, expected: { mode: scenario.mode, risk: scenario.expectedRisk }, observed, result: "PASS",
|
|
95
|
-
counts, source, timingsMs: Object.fromEntries(Object.entries(times).map(([name, values]) => [name, distribution(values)])) };
|
|
96
|
-
}
|
|
97
|
-
function stressQuota(sourceFiles, quota) {
|
|
98
|
-
const fixture = workspace(sourceFiles), start = performance.now();
|
|
99
|
-
try {
|
|
100
|
-
const task = fixture.init("bounded", scenarios[0]);
|
|
101
|
-
for (let i = 0; i < quota; i++) cancelFixture(fixture.project, reserve(fixture.project, task.id, request(task, `try-${i}`, quota)));
|
|
102
|
-
assert.throws(() => reserve(fixture.project, task.id, request(task, "over-budget", quota)), /budget reserved for bounded/i);
|
|
103
|
-
const after = status(fixture.project, task.id);
|
|
104
|
-
assert.equal(after.usage.dispatches, quota);
|
|
105
|
-
assert.equal(after.telemetry.attempts, 0);
|
|
106
|
-
assert.equal(after.active.length, 0);
|
|
107
|
-
assert.equal(after.deliveryBudget.protectedSlots, 4);
|
|
108
|
-
return { id: "repeated-reservation-exhaustion", result: "PASS", quota, reservations: after.usage.dispatches, rejectedAdditionalReservation: true,
|
|
109
|
-
quotaKind: "ordinary-delivery", totalDispatchLimit: after.limits.maxDispatches, protectedRemaining: after.deliveryBudget.remaining,
|
|
110
|
-
cancelledAttemptsRefunded: false, providerAttempts: 0, durationMs: rounded(performance.now() - start) };
|
|
111
|
-
} finally { fixture.dispose(); }
|
|
112
|
-
}
|
|
113
|
-
function stressParent(sourceFiles) {
|
|
114
|
-
const fixture = workspace(sourceFiles), start = performance.now();
|
|
115
|
-
try {
|
|
116
|
-
const parent = fixture.init("parent", scenarios[1]);
|
|
117
|
-
cancelFixture(fixture.project, reserve(fixture.project, parent.id, request(parent, "parent-integration", 4)));
|
|
118
|
-
const first = fixture.init("child-a", scenarios[0], parent.id);
|
|
119
|
-
for (let i = 0; i < 2; i++) cancelFixture(fixture.project, reserve(fixture.project, first.id, request(first, `child-a-${i}`, 3)));
|
|
120
|
-
const second = fixture.init("child-b", scenarios[0], parent.id);
|
|
121
|
-
cancelFixture(fixture.project, reserve(fixture.project, second.id, request(second, "child-b-0", 3)));
|
|
122
|
-
assert.throws(() => reserve(fixture.project, second.id, request(second, "child-b-over-parent", 3)), /budget reserved for parent/i);
|
|
123
|
-
const third = fixture.init("child-c", scenarios[0], parent.id);
|
|
124
|
-
assert.throws(() => reserve(fixture.project, third.id, request(third, "new-child-no-refund", 3)), /budget reserved for parent/i);
|
|
125
|
-
const aggregate = status(fixture.project, parent.id);
|
|
126
|
-
assert.equal(aggregate.usage.dispatches, 4);
|
|
127
|
-
assert.equal(aggregate.telemetry.attempts, 0);
|
|
128
|
-
assert.equal(json(taskPath(fixture.project, parent.id)).reviewBudget.used, 0);
|
|
129
|
-
return { id: "parent-child-aggregate-exhaustion", result: "PASS", parentQuota: 4, aggregateReservations: 4,
|
|
130
|
-
quotaKind: "ordinary-delivery", totalDispatchLimit: aggregate.limits.maxDispatches, protectedRemaining: aggregate.deliveryBudget.remaining,
|
|
131
|
-
ownParentReservations: 1, childReservations: { "child-a": 2, "child-b": 1, "child-c": 0 },
|
|
132
|
-
blockedExistingChildDespiteLocalCapacity: true, blockedNewChildDespiteLocalCapacity: true,
|
|
133
|
-
providerAttempts: 0, durationMs: rounded(performance.now() - start) };
|
|
134
|
-
} finally { fixture.dispose(); }
|
|
135
|
-
}
|
|
136
|
-
export function runBenchmark({ iterations = 3, sourceFiles = 40, stressQuota: quota = 6 } = {}) {
|
|
137
|
-
assert(!(process.env.CLAUDECODE || process.env.CLAUDE_CODE_ENTRYPOINT), "Run the synthetic framework benchmark outside Claude Code; it uses explicit fixture reconciliation");
|
|
138
|
-
assert(Number.isSafeInteger(iterations) && iterations >= 1 && iterations <= 10, "iterations must be 1..10");
|
|
139
|
-
assert(Number.isSafeInteger(sourceFiles) && sourceFiles >= 1 && sourceFiles <= 1000, "sourceFiles must be 1..1000");
|
|
140
|
-
assert(Number.isSafeInteger(quota) && quota >= 1 && quota <= 32, "stressQuota must be 1..32");
|
|
141
|
-
const start = performance.now();
|
|
142
|
-
const results = scenarios.map(scenario => scenarioRun(scenario, iterations, sourceFiles));
|
|
143
|
-
const stress = [stressQuota(sourceFiles, quota), stressParent(sourceFiles)];
|
|
144
|
-
return { schemaVersion: 1, benchmark: "nccgs-v15-framework-only", label, recordedAt: new Date().toISOString(), result: "PASS",
|
|
145
|
-
environment: { node: process.version, platform: process.platform, architecture: process.arch },
|
|
146
|
-
framework: { packageVersion: json(path.join(root, "package.json")).version, pipelineSha256: hash(fs.readFileSync(path.join(root, ".claude/nccgs/pipeline.json"))) },
|
|
147
|
-
parameters: { iterations, sourceFiles, stressQuota: quota },
|
|
148
|
-
scenarios: results, stress,
|
|
149
|
-
totals: { scenarioRuns: scenarios.length * iterations, stressCases: stress.length, providerAttempts: 0, frameworkDurationMs: rounded(performance.now() - start) },
|
|
150
|
-
evidenceBoundary: { evidenceKind: "synthetic-framework-fixtures", modelExecution: "NOT_RUN", gameExecution: "NOT_RUN", tokenCost: "NOT_MEASURED", productivityComparison: "NOT_MEASURED",
|
|
151
|
-
interpretation: "Timings measure local framework I/O and bookkeeping only; total duration also includes fixture setup and cleanup. Fixture classification and quota assertions do not establish natural-language classification accuracy, model quality or whole-workflow efficiency." } };
|
|
152
|
-
}
|
|
153
|
-
export function run(args) {
|
|
154
|
-
const accepted = new Set(["--out", "--iterations", "--source-files", "--stress-quota"]);
|
|
155
|
-
const options = {};
|
|
156
|
-
for (let i = 0; i < args.length; i += 2) {
|
|
157
|
-
assert(accepted.has(args[i]) && args[i + 1] && !args[i + 1].startsWith("--"), "Usage: benchmark-v15.mjs [--out FILE] [--iterations 1..10] [--source-files 1..1000] [--stress-quota 1..32]");
|
|
158
|
-
assert(options[args[i]] === undefined, `Duplicate option: ${args[i]}`);
|
|
159
|
-
options[args[i]] = args[i + 1];
|
|
160
|
-
}
|
|
161
|
-
const result = runBenchmark({ ...(options["--iterations"] ? { iterations: Number(options["--iterations"]) } : {}),
|
|
162
|
-
...(options["--source-files"] ? { sourceFiles: Number(options["--source-files"]) } : {}), ...(options["--stress-quota"] ? { stressQuota: Number(options["--stress-quota"]) } : {}) });
|
|
163
|
-
if (options["--out"]) {
|
|
164
|
-
const file = path.resolve(options["--out"]);
|
|
165
|
-
atomicJson(file, result);
|
|
166
|
-
return { result: result.result, label, output: file, totals: result.totals };
|
|
167
|
-
}
|
|
168
|
-
return result;
|
|
169
|
-
}
|
|
170
|
-
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
171
|
-
try { console.log(JSON.stringify(run(process.argv.slice(2)), null, 2)); }
|
|
172
|
-
catch (error) { console.error(error.message); process.exitCode = 1; }
|
|
173
|
-
}
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { performance } from "node:perf_hooks";
|
|
6
|
+
import { fileURLToPath } from "node:url";
|
|
7
|
+
import { atomicJson, hash, json } from "../.claude/nccgs/tools/policy.mjs";
|
|
8
|
+
import { initialize, record, snapshot, taskPath } from "../.claude/nccgs/tools/gates.mjs";
|
|
9
|
+
import { reserve, reconcile, status } from "../.claude/nccgs/tools/orchestration.mjs";
|
|
10
|
+
|
|
11
|
+
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
12
|
+
const label = "SYNTHETIC FRAMEWORK BENCHMARK — no provider/model or game execution";
|
|
13
|
+
export const scenarios = [
|
|
14
|
+
{ id: "tiny-fix", name: "Tiny local fix", mode: "implement", scope: "local", target: "fix", expectedRisk: "FAST", objective: "Fix a local display label alignment with a known expected value" },
|
|
15
|
+
{ id: "learning-prototype", name: "Learning prototype", mode: "implement", scope: "feature", target: "learning-prototype", expectedRisk: "STANDARD", objective: "Explore a bounded feedback hypothesis in a synthetic prototype" },
|
|
16
|
+
{ id: "alpha-ready-prototype", name: "Alpha-ready prototype", mode: "implement", scope: "feature", target: "alpha-ready", expectedRisk: "STANDARD", objective: "Prepare a prototype feature for a declared end-to-end readiness check" },
|
|
17
|
+
{ id: "cross-system-design", name: "Cross-system optimization and design plan", mode: "plan", scope: "cross-system", target: "production-change", expectedRisk: "CONTROLLED", objective: "Plan cross-system allocation and update-order optimization from a measured baseline" },
|
|
18
|
+
{ id: "large-system-audit", name: "Large feature and system audit", mode: "audit", scope: "cross-system", target: "production-change", expectedRisk: "CONTROLLED", objective: "Audit dependencies and acceptance coverage across a large feature's systems" }
|
|
19
|
+
];
|
|
20
|
+
const measure = action => {
|
|
21
|
+
const start = performance.now(), value = action();
|
|
22
|
+
return { value, ms: performance.now() - start };
|
|
23
|
+
};
|
|
24
|
+
const rounded = value => Math.round(value * 1000) / 1000;
|
|
25
|
+
function distribution(values) {
|
|
26
|
+
const sorted = [...values].sort((a, b) => a - b), mid = Math.floor(sorted.length / 2);
|
|
27
|
+
return { samples: sorted.length, min: rounded(sorted[0]), median: rounded(sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2), max: rounded(sorted.at(-1)), total: rounded(sorted.reduce((a, b) => a + b, 0)) };
|
|
28
|
+
}
|
|
29
|
+
function workspace(sourceFiles) {
|
|
30
|
+
const project = fs.mkdtempSync(path.join(os.tmpdir(), "nccgs-v15-benchmark-"));
|
|
31
|
+
function write(relative, value) {
|
|
32
|
+
const file = path.join(project, relative);
|
|
33
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
34
|
+
fs.writeFileSync(file, typeof value === "string" ? value : `${JSON.stringify(value, null, 2)}\n`);
|
|
35
|
+
}
|
|
36
|
+
for (const file of ["pipeline.json", "studio.json"]) write(`.claude/nccgs/${file}`, fs.readFileSync(path.join(root, ".claude/nccgs", file), "utf8"));
|
|
37
|
+
write(".nccgs/project.yaml", "schema_version: 2\nmodels:\n profile: claude-standalone\n");
|
|
38
|
+
write(".nccgs/execution.json", { schemaVersion: 1, name: "standalone", externalReview: { mode: "off" }, acceptanceRequired: false, requireObservedEffort: false, maxParallelWorkers: 2 });
|
|
39
|
+
for (let i = 0; i < sourceFiles; i++) write(`SyntheticSource/component-${i}.txt`, `Synthetic framework fixture component ${i}.\n${"Boundary and dependency data only.\n".repeat(24)}`);
|
|
40
|
+
return { project, write,
|
|
41
|
+
init(id, scenario, parentId = null) {
|
|
42
|
+
const relative = `.nccgs/intakes/${id}.json`;
|
|
43
|
+
write(relative, { schemaVersion: 1, objective: scenario.objective, mode: scenario.mode, scope: scenario.scope, target: scenario.target, readiness: "ready", systems: ["synthetic-component"], requiredChecks: ["fixture-acceptance"] });
|
|
44
|
+
return initialize(project, id, null, null, null, { intake: relative, parentId });
|
|
45
|
+
},
|
|
46
|
+
dispose() {
|
|
47
|
+
const resolved = path.resolve(project), expectedParent = path.resolve(os.tmpdir());
|
|
48
|
+
assert.equal(path.dirname(resolved), expectedParent);
|
|
49
|
+
assert.ok(path.basename(resolved).startsWith("nccgs-v15-benchmark-"));
|
|
50
|
+
fs.rmSync(resolved, { recursive: true, force: true });
|
|
51
|
+
} };
|
|
52
|
+
}
|
|
53
|
+
function request(task, key, maxDispatches = null) {
|
|
54
|
+
// The stress parameter counts ordinary delivery attempts; protected slots stay
|
|
55
|
+
// inside the actual total ceiling and must remain untouched by this fixture.
|
|
56
|
+
const protectedSlots = Object.values(task.contract.deliveryReserve ?? {}).reduce((a, b) => a + b, 0);
|
|
57
|
+
return { kind: task.mode === "implement" ? "implementation" : "analysis", agent: task.mode === "implement" ? "nccgs-gameplay-programmer" : "nccgs-task-scout",
|
|
58
|
+
requestKey: key, reason: "Synthetic reservation only; no worker/provider is invoked", writePaths: task.mode === "implement" ? ["SyntheticSource"] : [],
|
|
59
|
+
...(maxDispatches === null ? {} : { limits: { maxDispatches: maxDispatches + protectedSlots } }) };
|
|
60
|
+
}
|
|
61
|
+
function cancelFixture(project, dispatch) {
|
|
62
|
+
return reconcile(project, dispatch.id, { status: "cancelled", reason: "Synthetic framework reservation completed without dispatching a model", reference: "benchmark-v15: intentionally no provider invocation" });
|
|
63
|
+
}
|
|
64
|
+
function scenarioRun(scenario, iterations, sourceFiles) {
|
|
65
|
+
const times = { init: [], snapshot: [], reserve: [], report: [] }, counts = { initialized: 0, snapshots: 0, reservations: 0, reports: 0, providerAttempts: 0 };
|
|
66
|
+
let source = null, observed = null;
|
|
67
|
+
for (let i = 0; i < iterations; i++) {
|
|
68
|
+
const fixture = workspace(sourceFiles);
|
|
69
|
+
try {
|
|
70
|
+
const initialized = measure(() => fixture.init(scenario.id, scenario));
|
|
71
|
+
const task = initialized.value;
|
|
72
|
+
assert.equal(task.mode, scenario.mode); assert.equal(task.risk, scenario.expectedRisk);
|
|
73
|
+
assert.equal(task.schemaVersion, 2);
|
|
74
|
+
const expectedRoute = { FAST: { class: "lightweight", model: "sonnet", effort: "high" }, STANDARD: { class: "standard", model: "opus", effort: "high" }, CONTROLLED: { class: "deep", model: "opus", effort: "xhigh" } }[scenario.expectedRisk];
|
|
75
|
+
assert.deepEqual(task.implementationRoute, expectedRoute);
|
|
76
|
+
observed = { mode: task.mode, risk: task.risk, implementationRoute: task.implementationRoute };
|
|
77
|
+
times.init.push(initialized.ms); counts.initialized++;
|
|
78
|
+
const captured = measure(() => snapshot(fixture.project, task));
|
|
79
|
+
times.snapshot.push(captured.ms); counts.snapshots++;
|
|
80
|
+
source = { syntheticFiles: sourceFiles, snapshotFiles: captured.value.files, snapshotBytes: captured.value.bytes };
|
|
81
|
+
const reserved = measure(() => reserve(fixture.project, task.id, request(task, "measurement")));
|
|
82
|
+
times.reserve.push(reserved.ms); counts.reservations++;
|
|
83
|
+
cancelFixture(fixture.project, reserved.value);
|
|
84
|
+
const reported = measure(() => record(fixture.project, task.id, "report", { outcome: "blocked", summary: `${label}. Classification, snapshot, reservation and reporting were exercised; there is no runtime/model evidence.` }));
|
|
85
|
+
times.report.push(reported.ms); counts.reports++;
|
|
86
|
+
const usage = status(fixture.project, task.id);
|
|
87
|
+
assert.equal(usage.usage.dispatches, 1);
|
|
88
|
+
assert.equal(usage.telemetry.attempts, 0);
|
|
89
|
+
assert.equal(reported.value.reviewBudget.used, 0);
|
|
90
|
+
assert.equal(reported.value.stages.report.verificationState.verified, false);
|
|
91
|
+
assert.equal(reported.value.outcome, "blocked");
|
|
92
|
+
} finally { fixture.dispose(); }
|
|
93
|
+
}
|
|
94
|
+
return { id: scenario.id, name: scenario.name, expected: { mode: scenario.mode, risk: scenario.expectedRisk }, observed, result: "PASS",
|
|
95
|
+
counts, source, timingsMs: Object.fromEntries(Object.entries(times).map(([name, values]) => [name, distribution(values)])) };
|
|
96
|
+
}
|
|
97
|
+
function stressQuota(sourceFiles, quota) {
|
|
98
|
+
const fixture = workspace(sourceFiles), start = performance.now();
|
|
99
|
+
try {
|
|
100
|
+
const task = fixture.init("bounded", scenarios[0]);
|
|
101
|
+
for (let i = 0; i < quota; i++) cancelFixture(fixture.project, reserve(fixture.project, task.id, request(task, `try-${i}`, quota)));
|
|
102
|
+
assert.throws(() => reserve(fixture.project, task.id, request(task, "over-budget", quota)), /budget reserved for bounded/i);
|
|
103
|
+
const after = status(fixture.project, task.id);
|
|
104
|
+
assert.equal(after.usage.dispatches, quota);
|
|
105
|
+
assert.equal(after.telemetry.attempts, 0);
|
|
106
|
+
assert.equal(after.active.length, 0);
|
|
107
|
+
assert.equal(after.deliveryBudget.protectedSlots, 4);
|
|
108
|
+
return { id: "repeated-reservation-exhaustion", result: "PASS", quota, reservations: after.usage.dispatches, rejectedAdditionalReservation: true,
|
|
109
|
+
quotaKind: "ordinary-delivery", totalDispatchLimit: after.limits.maxDispatches, protectedRemaining: after.deliveryBudget.remaining,
|
|
110
|
+
cancelledAttemptsRefunded: false, providerAttempts: 0, durationMs: rounded(performance.now() - start) };
|
|
111
|
+
} finally { fixture.dispose(); }
|
|
112
|
+
}
|
|
113
|
+
function stressParent(sourceFiles) {
|
|
114
|
+
const fixture = workspace(sourceFiles), start = performance.now();
|
|
115
|
+
try {
|
|
116
|
+
const parent = fixture.init("parent", scenarios[1]);
|
|
117
|
+
cancelFixture(fixture.project, reserve(fixture.project, parent.id, request(parent, "parent-integration", 4)));
|
|
118
|
+
const first = fixture.init("child-a", scenarios[0], parent.id);
|
|
119
|
+
for (let i = 0; i < 2; i++) cancelFixture(fixture.project, reserve(fixture.project, first.id, request(first, `child-a-${i}`, 3)));
|
|
120
|
+
const second = fixture.init("child-b", scenarios[0], parent.id);
|
|
121
|
+
cancelFixture(fixture.project, reserve(fixture.project, second.id, request(second, "child-b-0", 3)));
|
|
122
|
+
assert.throws(() => reserve(fixture.project, second.id, request(second, "child-b-over-parent", 3)), /budget reserved for parent/i);
|
|
123
|
+
const third = fixture.init("child-c", scenarios[0], parent.id);
|
|
124
|
+
assert.throws(() => reserve(fixture.project, third.id, request(third, "new-child-no-refund", 3)), /budget reserved for parent/i);
|
|
125
|
+
const aggregate = status(fixture.project, parent.id);
|
|
126
|
+
assert.equal(aggregate.usage.dispatches, 4);
|
|
127
|
+
assert.equal(aggregate.telemetry.attempts, 0);
|
|
128
|
+
assert.equal(json(taskPath(fixture.project, parent.id)).reviewBudget.used, 0);
|
|
129
|
+
return { id: "parent-child-aggregate-exhaustion", result: "PASS", parentQuota: 4, aggregateReservations: 4,
|
|
130
|
+
quotaKind: "ordinary-delivery", totalDispatchLimit: aggregate.limits.maxDispatches, protectedRemaining: aggregate.deliveryBudget.remaining,
|
|
131
|
+
ownParentReservations: 1, childReservations: { "child-a": 2, "child-b": 1, "child-c": 0 },
|
|
132
|
+
blockedExistingChildDespiteLocalCapacity: true, blockedNewChildDespiteLocalCapacity: true,
|
|
133
|
+
providerAttempts: 0, durationMs: rounded(performance.now() - start) };
|
|
134
|
+
} finally { fixture.dispose(); }
|
|
135
|
+
}
|
|
136
|
+
export function runBenchmark({ iterations = 3, sourceFiles = 40, stressQuota: quota = 6 } = {}) {
|
|
137
|
+
assert(!(process.env.CLAUDECODE || process.env.CLAUDE_CODE_ENTRYPOINT), "Run the synthetic framework benchmark outside Claude Code; it uses explicit fixture reconciliation");
|
|
138
|
+
assert(Number.isSafeInteger(iterations) && iterations >= 1 && iterations <= 10, "iterations must be 1..10");
|
|
139
|
+
assert(Number.isSafeInteger(sourceFiles) && sourceFiles >= 1 && sourceFiles <= 1000, "sourceFiles must be 1..1000");
|
|
140
|
+
assert(Number.isSafeInteger(quota) && quota >= 1 && quota <= 32, "stressQuota must be 1..32");
|
|
141
|
+
const start = performance.now();
|
|
142
|
+
const results = scenarios.map(scenario => scenarioRun(scenario, iterations, sourceFiles));
|
|
143
|
+
const stress = [stressQuota(sourceFiles, quota), stressParent(sourceFiles)];
|
|
144
|
+
return { schemaVersion: 1, benchmark: "nccgs-v15-framework-only", label, recordedAt: new Date().toISOString(), result: "PASS",
|
|
145
|
+
environment: { node: process.version, platform: process.platform, architecture: process.arch },
|
|
146
|
+
framework: { packageVersion: json(path.join(root, "package.json")).version, pipelineSha256: hash(fs.readFileSync(path.join(root, ".claude/nccgs/pipeline.json"))) },
|
|
147
|
+
parameters: { iterations, sourceFiles, stressQuota: quota },
|
|
148
|
+
scenarios: results, stress,
|
|
149
|
+
totals: { scenarioRuns: scenarios.length * iterations, stressCases: stress.length, providerAttempts: 0, frameworkDurationMs: rounded(performance.now() - start) },
|
|
150
|
+
evidenceBoundary: { evidenceKind: "synthetic-framework-fixtures", modelExecution: "NOT_RUN", gameExecution: "NOT_RUN", tokenCost: "NOT_MEASURED", productivityComparison: "NOT_MEASURED",
|
|
151
|
+
interpretation: "Timings measure local framework I/O and bookkeeping only; total duration also includes fixture setup and cleanup. Fixture classification and quota assertions do not establish natural-language classification accuracy, model quality or whole-workflow efficiency." } };
|
|
152
|
+
}
|
|
153
|
+
export function run(args) {
|
|
154
|
+
const accepted = new Set(["--out", "--iterations", "--source-files", "--stress-quota"]);
|
|
155
|
+
const options = {};
|
|
156
|
+
for (let i = 0; i < args.length; i += 2) {
|
|
157
|
+
assert(accepted.has(args[i]) && args[i + 1] && !args[i + 1].startsWith("--"), "Usage: benchmark-v15.mjs [--out FILE] [--iterations 1..10] [--source-files 1..1000] [--stress-quota 1..32]");
|
|
158
|
+
assert(options[args[i]] === undefined, `Duplicate option: ${args[i]}`);
|
|
159
|
+
options[args[i]] = args[i + 1];
|
|
160
|
+
}
|
|
161
|
+
const result = runBenchmark({ ...(options["--iterations"] ? { iterations: Number(options["--iterations"]) } : {}),
|
|
162
|
+
...(options["--source-files"] ? { sourceFiles: Number(options["--source-files"]) } : {}), ...(options["--stress-quota"] ? { stressQuota: Number(options["--stress-quota"]) } : {}) });
|
|
163
|
+
if (options["--out"]) {
|
|
164
|
+
const file = path.resolve(options["--out"]);
|
|
165
|
+
atomicJson(file, result);
|
|
166
|
+
return { result: result.result, label, output: file, totals: result.totals };
|
|
167
|
+
}
|
|
168
|
+
return result;
|
|
169
|
+
}
|
|
170
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
171
|
+
try { console.log(JSON.stringify(run(process.argv.slice(2)), null, 2)); }
|
|
172
|
+
catch (error) { console.error(error.message); process.exitCode = 1; }
|
|
173
|
+
}
|