@event4u/agent-config 16.1.0 → 16.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.augment-plugin/marketplace.json +3 -3
- package/.augment-plugin/plugin.json +1 -1
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +237 -248
- package/dist/agent-src/commands/roadmap/process-full.md +1 -1
- package/dist/agent-src/commands/work.md +4 -0
- package/dist/agent-src/contexts/execution/autonomy-mechanics.md +4 -8
- package/dist/agent-src/rules/ui-audit-gate.md +1 -2
- package/dist/agent-src/skills/accessibility-auditor/data/aria-patterns.csv +1 -1
- package/dist/agent-src/skills/accessibility-auditor/data/manifest.json +2 -2
- package/dist/agent-src/skills/design-review/SKILL.md +30 -4
- package/dist/agent-src/skills/design-review/references/verification-automation.md +32 -0
- package/dist/agent-src/skills/existing-ui-audit/SKILL.md +16 -1
- package/dist/agent-src/skills/existing-ui-audit/references/component-taxonomy-detection.md +56 -0
- package/dist/agent-src/skills/markitdown/SKILL.md +4 -0
- package/dist/agent-src/skills/playwright-testing/SKILL.md +30 -6
- package/dist/agent-src/skills/ui-component-architect/SKILL.md +23 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/roadmaps.md +30 -0
- package/dist/agent-src/templates/scripts/work_engine/directives/ui/app_spec.ts +10 -4
- package/dist/agent-src/templates/scripts/work_engine/directives/ui/apply.ts +260 -21
- package/dist/agent-src/templates/scripts/work_engine/directives/ui/audit.ts +22 -3
- package/dist/agent-src/templates/scripts/work_engine/directives/ui/scaffold.ts +6 -1
- package/dist/agent-src/templates/scripts/work_engine/state.ts +57 -7
- package/dist/agent-src/templates/scripts/work_engine/taxonomy/detect.ts +500 -0
- package/dist/agent-src/templates/scripts/work_engine/taxonomy/place.ts +183 -0
- package/dist/cli/python/knowledge_ingest.js +1 -1
- package/dist/cli/python/workspace_drive.js +1 -1
- package/dist/cli-delegate/{chunk-S3VFXMTM.js → chunk-4XFP2Y3E.js} +1 -1
- package/dist/cli-delegate/{chunk-HDVVGPQW.js → chunk-7XGBX6ZH.js} +471 -387
- package/dist/cli-delegate/{chunk-GJCBB2UG.js → chunk-FP36SWJD.js} +432 -144
- package/dist/cli-delegate/{chunk-UNCRJBT3.js → chunk-IGD6ERX6.js} +31 -0
- package/dist/cli-delegate/{chunk-FXYUYU4N.js → chunk-JNMVLWLR.js} +42 -1
- package/dist/cli-delegate/{chunk-QJWAUMJW.js → chunk-YJ72UIAK.js} +730 -15
- package/dist/cli-delegate/cmd_conformance.js +902 -782
- package/dist/cli-delegate/cmd_doctor.js +2 -2
- package/dist/cli-delegate/cmd_export.js +2 -2
- package/dist/cli-delegate/cmd_handoff.js +4 -4
- package/dist/cli-delegate/cmd_memory_get.js +1 -1
- package/dist/cli-delegate/cmd_prune.js +2 -2
- package/dist/cli-delegate/cmd_route_audit.js +1 -1
- package/dist/cli-delegate/cmd_session_recycle.js +2 -2
- package/dist/cli-delegate/cmd_settings_migrate.js +2 -2
- package/dist/cli-delegate/cmd_settings_set.js +1 -1
- package/dist/cli-delegate/cmd_sync.js +2 -2
- package/dist/cli-delegate/cmd_uninstall.js +2 -2
- package/dist/cli-delegate/cmd_validate.js +2 -2
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +10 -10
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +6666 -2581
- package/dist/hooks/dispatch.sha256 +1 -0
- package/dist/install/install.mjs +848 -733
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +2747 -1510
- package/dist/server/schemas/settings.js +23 -0
- package/dist/ui/assets/index-Cwq4d0hX.js +23 -0
- package/dist/ui/assets/index-Cyjy8bWO.css +1 -0
- package/dist/ui/index.html +2 -2
- package/docs/contracts/settings-classes.md +6 -4
- package/package.json +4 -4
- package/src/agent-src/templates/scripts/work_engine/directives/ui/app_spec.ts +10 -4
- package/src/agent-src/templates/scripts/work_engine/directives/ui/apply.ts +260 -21
- package/src/agent-src/templates/scripts/work_engine/directives/ui/audit.ts +22 -3
- package/src/agent-src/templates/scripts/work_engine/directives/ui/scaffold.ts +6 -1
- package/src/agent-src/templates/scripts/work_engine/state.ts +57 -7
- package/src/agent-src/templates/scripts/work_engine/taxonomy/detect.ts +500 -0
- package/src/agent-src/templates/scripts/work_engine/taxonomy/place.ts +183 -0
- package/src/config/agent-settings.template.yml +29 -0
- package/src/config/ci-local-parity.yml +18 -0
- package/src/config/continuity-surface.json +2 -2
- package/src/config/gate-coverage.yml +234 -3
- package/src/config/gate-violation-baselines.json +7 -3
- package/src/config/hook-latency-budget.json +21 -0
- package/src/config/hook-token-budget.json +2 -2
- package/src/config/metric-registry.yml +114 -0
- package/src/config/placeholder-drift-budget.json +10 -0
- package/src/config/routing-coverage-seed.json +18 -4
- package/src/scripts/_cli/cmd_doctor.ts +5 -19
- package/src/scripts/_cli/doctor_execution.ts +100 -17
- package/src/scripts/_cli/doctor_network_posture.ts +226 -0
- package/src/scripts/_lib/body_portable.ts +102 -0
- package/src/scripts/_lib/cc_transcript.ts +132 -1
- package/src/scripts/_lib/concern_sla_window.ts +200 -0
- package/src/scripts/_lib/forge_protection.ts +114 -0
- package/src/scripts/_lib/forge_reader.ts +558 -0
- package/src/scripts/_lib/host_env_write.ts +111 -0
- package/src/scripts/_lib/llm_proposer_transport.ts +14 -4
- package/src/scripts/_lib/obligations.ts +119 -8
- package/src/scripts/_lib/probe_inputs.ts +271 -0
- package/src/scripts/_lib/read_surface_scan.ts +322 -0
- package/src/scripts/_lib/release_highlights.ts +2 -1
- package/src/scripts/_lib/retrieval_sanitize.ts +77 -16
- package/src/scripts/_lib/shell_write_shapes.ts +108 -0
- package/src/scripts/_lib/structural_hiding.ts +622 -0
- package/src/scripts/_lib/turn_end_refusals.ts +525 -4
- package/src/scripts/_lib/turn_end_transcript.ts +216 -0
- package/src/scripts/_lib/verification_command.ts +324 -0
- package/src/scripts/_lib/verification_evidence.ts +647 -0
- package/src/scripts/_lib/verify_clause.ts +280 -0
- package/src/scripts/ai_team/team_dispatch.ts +4 -0
- package/src/scripts/before_complete_hook.ts +340 -23
- package/src/scripts/bench_hook_latency.ts +412 -5
- package/src/scripts/cache_realization_report.ts +61 -1
- package/src/scripts/check_argument_hint.ts +287 -0
- package/src/scripts/check_claims.ts +125 -1
- package/src/scripts/check_enforcement_coverage.ts +38 -9
- package/src/scripts/check_enforcement_matrix.ts +245 -3
- package/src/scripts/check_finding_dispositions.ts +22 -3
- package/src/scripts/check_generator_sync.ts +71 -0
- package/src/scripts/check_host_docs_digest.ts +786 -0
- package/src/scripts/check_host_format_column.ts +559 -0
- package/src/scripts/check_input_references.ts +228 -0
- package/src/scripts/check_inputs_parity.ts +235 -0
- package/src/scripts/check_kernel_edit_ratified.ts +47 -5
- package/src/scripts/check_kill_switch_table.ts +294 -0
- package/src/scripts/check_placeholder_drift.ts +221 -0
- package/src/scripts/check_read_surface_coverage.ts +515 -0
- package/src/scripts/check_release_holds.ts +2 -1
- package/src/scripts/check_routing_coverage.ts +378 -11
- package/src/scripts/check_static_parity.ts +17 -0
- package/src/scripts/ci_settle.ts +145 -19
- package/src/scripts/closure_scan.ts +115 -1
- package/src/scripts/compile_hook_manifest.ts +60 -16
- package/src/scripts/external_sources_denylist.json +29 -4
- package/src/scripts/hook_manifest.json +1 -1
- package/src/scripts/hook_manifest.yaml +42 -5
- package/src/scripts/hooks/block_config_weakening.ts +229 -4
- package/src/scripts/hooks/block_kernel_rule_writes.ts +9 -78
- package/src/scripts/hooks/block_plumbing_writes.ts +314 -0
- package/src/scripts/hooks/bundle_integrity.ts +291 -0
- package/src/scripts/hooks/chain_nudge_hook.ts +1 -2
- package/src/scripts/hooks/code_graph_context_hook.ts +1 -2
- package/src/scripts/hooks/comment_discipline_hook.ts +1 -3
- package/src/scripts/hooks/concern_registry.ts +2 -0
- package/src/scripts/hooks/concern_timings.ts +94 -0
- package/src/scripts/hooks/delegation_nudge_hook.ts +1 -2
- package/src/scripts/hooks/design_pass_hook.ts +100 -9
- package/src/scripts/hooks/design_slop_hook.ts +1 -2
- package/src/scripts/hooks/dispatch_hook.ts +25 -19
- package/src/scripts/hooks/edit_shape_hook.ts +1 -2
- package/src/scripts/hooks/end_review_nudge_hook.ts +1 -1
- package/src/scripts/hooks/evidence_independence.ts +1 -2
- package/src/scripts/hooks/exit_codes.ts +128 -0
- package/src/scripts/hooks/host_lowering.json +1 -0
- package/src/scripts/hooks/host_lowering.ts +128 -3
- package/src/scripts/hooks/host_lowering.yaml +262 -46
- package/src/scripts/hooks/host_semantics.ts +30 -2
- package/src/scripts/hooks/injection_budget.ts +9 -2
- package/src/scripts/hooks/interruption_ledger_hook.ts +1 -2
- package/src/scripts/hooks/journal_record_hook.ts +1 -2
- package/src/scripts/hooks/memory_recall_hook.ts +1 -2
- package/src/scripts/hooks/obligation_settle_hook.ts +151 -5
- package/src/scripts/hooks/one_question_per_ask_hook.ts +25 -4
- package/src/scripts/hooks/orchestration_record_hook.ts +1 -2
- package/src/scripts/hooks/permission_gate.ts +1 -2
- package/src/scripts/hooks/push_settle_hook.ts +1 -3
- package/src/scripts/hooks/replay_hook.ts +2 -1
- package/src/scripts/hooks/reread_guard_hook.ts +1 -2
- package/src/scripts/hooks/review_baseline_hook.ts +1 -2
- package/src/scripts/hooks/rtk_wrap_hook.ts +1 -2
- package/src/scripts/hooks/rule_inject_hook.ts +8 -5
- package/src/scripts/hooks/run_continuation_hook.ts +13 -15
- package/src/scripts/hooks/skill_route_hook.ts +1 -2
- package/src/scripts/hooks/source_first_gate_hook.ts +1 -2
- package/src/scripts/hooks/spawn_guard_shadow_hook.ts +1 -2
- package/src/scripts/hooks/subagent_ledger_hook.ts +1 -2
- package/src/scripts/hooks/suggestion_capture_hook.ts +1 -1
- package/src/scripts/hooks/table_fingerprint.ts +49 -0
- package/src/scripts/hooks/telemetry_flush_hook.ts +1 -2
- package/src/scripts/hooks/telemetry_usage_hook.ts +1 -2
- package/src/scripts/hooks/tool_result_bytes_hook.ts +93 -9
- package/src/scripts/hooks/turn_end_gate_hook.ts +337 -324
- package/src/scripts/hooks/ui_route_nudge_hook.ts +1 -2
- package/src/scripts/install.ts +3 -4
- package/src/scripts/lint_deny_text.ts +419 -0
- package/src/scripts/lint_exit_codes.ts +262 -0
- package/src/scripts/lint_hook_manifest.ts +55 -0
- package/src/scripts/lint_roadmap_blockers.ts +78 -12
- package/src/scripts/lint_skill_descriptions.ts +1 -1
- package/src/scripts/lint_skill_scripts_readonly.ts +1 -1
- package/src/scripts/mcp_server/prompts.ts +142 -1
- package/src/scripts/measure_skill_ranker_baseline.ts +305 -51
- package/src/scripts/measure_turn_end_gate.ts +193 -0
- package/src/scripts/memory_lookup.ts +1 -1
- package/src/scripts/prepack-check.mjs +53 -0
- package/src/scripts/release_publication.ts +13 -1
- package/src/scripts/report_invocation_surface.ts +383 -0
- package/src/scripts/report_obligation_carriers.ts +699 -0
- package/src/scripts/report_obligation_writer_split.ts +236 -0
- package/src/scripts/report_skill_menu_census.ts +208 -0
- package/src/scripts/report_standing_payload_by_host.ts +1 -1
- package/src/scripts/roadmap_verify_share.ts +213 -0
- package/src/scripts/schemas/command.schema.json +44 -0
- package/src/scripts/schemas/review-findings.schema.json +4 -0
- package/src/scripts/schemas/skill.schema.json +44 -0
- package/src/scripts/self_review_gate.ts +359 -10
- package/src/scripts/semantic_parity_census.ts +314 -0
- package/src/scripts/session_eol_report.ts +54 -1
- package/src/scripts/sweep_dead_scan_roots.ts +1 -1
- package/src/scripts/trigger_eval_grandfather.json +0 -2
- package/src/scripts/trigger_eval_rotation.ts +122 -27
- package/src/scripts/ui_conformance_probe.ts +156 -4
- package/src/scripts/update_prices.ts +85 -3
- package/src/scripts/write_bundle_digest.mjs +74 -0
- package/src/server/schemas/settings.ts +31 -0
- package/src/templates/consumer-settings/ONBOARDING.md +1 -0
- package/dist/ui/assets/index-DH1xc62v.js +0 -29
- package/dist/ui/assets/index-jSfMW8_t.css +0 -1
- /package/dist/cli-delegate/{chunk-YUM5KURG.js → chunk-7RQP4UDH.js} +0 -0
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* The semantic-parity census — which skill frontmatter semantics survive the
|
|
4
|
+
* MCP-lite carrier, and which are dropped on the way to a host.
|
|
5
|
+
*
|
|
6
|
+
* road-to-semantic-parity-before-off-menu, Phase 1.
|
|
7
|
+
*
|
|
8
|
+
* THE CARRIER THIS MEASURES
|
|
9
|
+
*
|
|
10
|
+
* Exactly one: `ContentEntry` in `src/cli/mcp/content.ts`. That scope is in the
|
|
11
|
+
* report header on purpose. A different delivery path drops a different set, so
|
|
12
|
+
* a second carrier is a second report rather than an edit to this one — merging
|
|
13
|
+
* two field sets into one list would leave neither denominator recoverable, and
|
|
14
|
+
* a report whose scope is not stated in the report gets quoted as general.
|
|
15
|
+
*
|
|
16
|
+
* THE CLASSIFICATION
|
|
17
|
+
*
|
|
18
|
+
* Every property the skill frontmatter schema declares falls into exactly one
|
|
19
|
+
* of three columns, decided by what the carrier's builder actually reads rather
|
|
20
|
+
* than by what its field comments say:
|
|
21
|
+
*
|
|
22
|
+
* - `carried` — the builder reads the frontmatter key and the value reaches
|
|
23
|
+
* `ContentEntry` whole.
|
|
24
|
+
* - `partial` — the builder reads the key but transports only part of the
|
|
25
|
+
* value. `triggers` is the only such key: `triggerText()` keeps
|
|
26
|
+
* `INDEXED_TRIGGER_KEYS` (`keyword`, `phrase`) and silently
|
|
27
|
+
* drops `file_pattern`, `path_prefix`, `command` and `reason`.
|
|
28
|
+
* Classing it `carried` would make a path-triggered skill look
|
|
29
|
+
* portable when its routing does not travel at all.
|
|
30
|
+
* - `dropped` — the builder never reads the key, so the semantic does not
|
|
31
|
+
* leave the disk.
|
|
32
|
+
*
|
|
33
|
+
* `carried` and `partial` are derived from the carrier source (the constants in
|
|
34
|
+
* `_lib/body_portable.ts` mirror `buildEntry` and `triggerText`), never from
|
|
35
|
+
* the schema. The
|
|
36
|
+
* `dropped` column is the remainder, so the three are total and disjoint by
|
|
37
|
+
* construction and a new schema property lands in `dropped` until someone
|
|
38
|
+
* teaches the carrier to read it — which is the failure direction this census
|
|
39
|
+
* exists to make visible.
|
|
40
|
+
*
|
|
41
|
+
* NOT IN THE CLASSIFICATION: `body`, `kind`, `mime_type` and `uri` are
|
|
42
|
+
* `ContentEntry` fields with no frontmatter key behind them — the body is the
|
|
43
|
+
* file minus its frontmatter, and the other three are derived from where the
|
|
44
|
+
* file sits. They are carrier fields, not declared semantics, so counting them
|
|
45
|
+
* as `carried` would inflate the numerator with things no skill declares.
|
|
46
|
+
*
|
|
47
|
+
* REPRODUCIBILITY
|
|
48
|
+
*
|
|
49
|
+
* The report pins the commit it was taken at and carries no wall-clock date, so
|
|
50
|
+
* a re-run at the same commit is byte-equal. Nothing machine-dependent is
|
|
51
|
+
* computed into it.
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
import { execFileSync } from 'node:child_process';
|
|
55
|
+
import fs from 'node:fs';
|
|
56
|
+
import path from 'node:path';
|
|
57
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
58
|
+
|
|
59
|
+
import { load as yamlLoad } from 'js-yaml';
|
|
60
|
+
|
|
61
|
+
import { CARRIED_KEYS, PARTIAL_KEYS, classifyPortability } from './_lib/body_portable.js';
|
|
62
|
+
|
|
63
|
+
const _HERE = fileURLToPath(import.meta.url);
|
|
64
|
+
export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
65
|
+
|
|
66
|
+
/** Where the schema that declares the semantics lives. */
|
|
67
|
+
const SCHEMA_REL = path.join('src', 'scripts', 'schemas', 'skill.schema.json');
|
|
68
|
+
|
|
69
|
+
/** Where the corpus lives. */
|
|
70
|
+
const SKILLS_REL = path.join('src', 'skills');
|
|
71
|
+
|
|
72
|
+
/** Default report location. */
|
|
73
|
+
const DEFAULT_OUT = path.join('agents', 'evidence', 'analysis', 'semantic-parity-census.md');
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The carrier constants live with the predicate in `_lib/body_portable.ts` so
|
|
77
|
+
* that the census and the eligibility check cannot drift apart: one source of
|
|
78
|
+
* truth, read here and there.
|
|
79
|
+
*/
|
|
80
|
+
export { CARRIED_KEYS, PARTIAL_KEYS };
|
|
81
|
+
|
|
82
|
+
export type Column = 'carried' | 'partial' | 'dropped';
|
|
83
|
+
|
|
84
|
+
export interface FieldRow {
|
|
85
|
+
readonly field: string;
|
|
86
|
+
readonly column: Column;
|
|
87
|
+
/** For `partial`, the sub-keys that travel. Empty otherwise. */
|
|
88
|
+
readonly travels: readonly string[];
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export interface SkillRow {
|
|
92
|
+
/** Directory name under `src/skills/`. */
|
|
93
|
+
readonly skill: string;
|
|
94
|
+
/** Declared keys that are `dropped`. */
|
|
95
|
+
readonly dropped: readonly string[];
|
|
96
|
+
/** Declared keys that are `partial`. */
|
|
97
|
+
readonly partial: readonly string[];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export interface Census {
|
|
101
|
+
readonly fields: readonly FieldRow[];
|
|
102
|
+
readonly skills: readonly SkillRow[];
|
|
103
|
+
/** Per dropped-or-partial field: how many skills declare it. */
|
|
104
|
+
readonly perField: Readonly<Record<string, number>>;
|
|
105
|
+
/** Total `SKILL.md` files read. */
|
|
106
|
+
readonly scanned: number;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Classify every schema property into exactly one column. */
|
|
110
|
+
export function classify(schemaProperties: readonly string[]): FieldRow[] {
|
|
111
|
+
const rows: FieldRow[] = [];
|
|
112
|
+
for (const field of [...schemaProperties].sort()) {
|
|
113
|
+
if (CARRIED_KEYS.includes(field)) {
|
|
114
|
+
rows.push({ field, column: 'carried', travels: [] });
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
const travels = PARTIAL_KEYS[field];
|
|
118
|
+
if (travels !== undefined) {
|
|
119
|
+
rows.push({ field, column: 'partial', travels });
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
rows.push({ field, column: 'dropped', travels: [] });
|
|
123
|
+
}
|
|
124
|
+
return rows;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Parsed top-level frontmatter of one `SKILL.md`, or `{}` when it carries none. */
|
|
128
|
+
export function frontmatterRecord(text: string): Record<string, unknown> {
|
|
129
|
+
const m = /^---\n([\s\S]*?)\n---/.exec(text);
|
|
130
|
+
if (m === null) return {};
|
|
131
|
+
let doc: unknown;
|
|
132
|
+
try {
|
|
133
|
+
doc = yamlLoad(m[1]!);
|
|
134
|
+
} catch {
|
|
135
|
+
return {};
|
|
136
|
+
}
|
|
137
|
+
if (doc === null || typeof doc !== 'object' || Array.isArray(doc)) return {};
|
|
138
|
+
return doc as Record<string, unknown>;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export function census(root: string): Census {
|
|
142
|
+
const schemaPath = path.join(root, SCHEMA_REL);
|
|
143
|
+
const schema = JSON.parse(fs.readFileSync(schemaPath, 'utf8')) as {
|
|
144
|
+
properties?: Record<string, unknown>;
|
|
145
|
+
};
|
|
146
|
+
const fields = classify(Object.keys(schema.properties ?? {}));
|
|
147
|
+
|
|
148
|
+
const skillsDir = path.join(root, SKILLS_REL);
|
|
149
|
+
const dirs = fs.existsSync(skillsDir)
|
|
150
|
+
? fs
|
|
151
|
+
.readdirSync(skillsDir, { withFileTypes: true })
|
|
152
|
+
.filter((d) => d.isDirectory())
|
|
153
|
+
.map((d) => d.name)
|
|
154
|
+
.sort()
|
|
155
|
+
: [];
|
|
156
|
+
|
|
157
|
+
const skills: SkillRow[] = [];
|
|
158
|
+
const perField: Record<string, number> = {};
|
|
159
|
+
let scanned = 0;
|
|
160
|
+
|
|
161
|
+
for (const dir of dirs) {
|
|
162
|
+
const file = path.join(skillsDir, dir, 'SKILL.md');
|
|
163
|
+
if (!fs.existsSync(file)) continue;
|
|
164
|
+
scanned += 1;
|
|
165
|
+
// The per-skill pass calls the PREDICATE rather than re-deriving the
|
|
166
|
+
// classification from the schema. An earlier version looked the key up
|
|
167
|
+
// in the schema-derived column map and skipped what it did not find,
|
|
168
|
+
// which made the census fail OPEN on an unknown key while the predicate
|
|
169
|
+
// failed CLOSED on the same key — two numbers presented as one
|
|
170
|
+
// measurement, disagreeing on exactly the case the predicate's own
|
|
171
|
+
// documentation calls its most important. Sharing the function removes
|
|
172
|
+
// the divergence class instead of correcting this instance of it.
|
|
173
|
+
const { dropped, partial } = classifyPortability(
|
|
174
|
+
frontmatterRecord(fs.readFileSync(file, 'utf8')),
|
|
175
|
+
);
|
|
176
|
+
for (const key of [...dropped, ...partial]) {
|
|
177
|
+
perField[key] = (perField[key] ?? 0) + 1;
|
|
178
|
+
}
|
|
179
|
+
skills.push({ skill: dir, dropped, partial });
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return { fields, skills, perField, scanned };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function pin(root: string): { sha: string; date: string } {
|
|
186
|
+
const git = (args: string[]): string =>
|
|
187
|
+
execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim();
|
|
188
|
+
try {
|
|
189
|
+
return { sha: git(['rev-parse', 'HEAD']), date: git(['show', '-s', '--format=%cI', 'HEAD']) };
|
|
190
|
+
} catch {
|
|
191
|
+
return { sha: 'unknown', date: 'unknown' };
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export function render(c: Census, p: { sha: string; date: string }): string {
|
|
196
|
+
const lines: string[] = [];
|
|
197
|
+
lines.push('<!-- evidence-type: analysis -->');
|
|
198
|
+
lines.push(`<!-- semantic-parity-census: v1 | commit: ${p.sha} | commit-date: ${p.date} -->`);
|
|
199
|
+
lines.push('');
|
|
200
|
+
lines.push('# Semantic-parity census — MCP-lite `ContentEntry`');
|
|
201
|
+
lines.push('');
|
|
202
|
+
lines.push(
|
|
203
|
+
'Emitted by `src/scripts/semantic_parity_census.ts`. **Carrier measured: `ContentEntry`**',
|
|
204
|
+
);
|
|
205
|
+
lines.push(
|
|
206
|
+
'in `src/cli/mcp/content.ts` — one delivery path, MCP-lite. A different path drops a',
|
|
207
|
+
);
|
|
208
|
+
lines.push(
|
|
209
|
+
'different set; a second carrier is a second report, never an edit to this one. Read the',
|
|
210
|
+
);
|
|
211
|
+
lines.push("script's module header for the classification before reading a number here.");
|
|
212
|
+
lines.push('');
|
|
213
|
+
lines.push('This report states coverage only. It marks no skill, excludes none, and is not a');
|
|
214
|
+
lines.push('verdict about which skills matter.');
|
|
215
|
+
lines.push('');
|
|
216
|
+
lines.push(`- **Commit pin:** \`${p.sha}\` (${p.date})`);
|
|
217
|
+
lines.push(`- **Schema:** \`${SCHEMA_REL}\``);
|
|
218
|
+
lines.push(`- **Corpus:** \`${SKILLS_REL}/*/SKILL.md\` — ${String(c.scanned)} file(s)`);
|
|
219
|
+
lines.push('');
|
|
220
|
+
|
|
221
|
+
const counts: Record<Column, number> = { carried: 0, partial: 0, dropped: 0 };
|
|
222
|
+
for (const r of c.fields) counts[r.column] += 1;
|
|
223
|
+
|
|
224
|
+
lines.push('## Field coverage');
|
|
225
|
+
lines.push('');
|
|
226
|
+
lines.push(
|
|
227
|
+
`${String(c.fields.length)} declared propert${c.fields.length === 1 ? 'y' : 'ies'}, each in exactly one column: ` +
|
|
228
|
+
`${String(counts.carried)} carried, ${String(counts.partial)} partial, ${String(counts.dropped)} dropped.`,
|
|
229
|
+
);
|
|
230
|
+
lines.push('');
|
|
231
|
+
lines.push('| field | carried | partial | dropped | travels |');
|
|
232
|
+
lines.push('|---|---|---|---|---|');
|
|
233
|
+
for (const r of c.fields) {
|
|
234
|
+
const mark = (col: Column): string => (r.column === col ? 'x' : '');
|
|
235
|
+
const travels = r.travels.length ? r.travels.map((t) => `\`${t}\``).join(', ') : '';
|
|
236
|
+
lines.push(
|
|
237
|
+
`| \`${r.field}\` | ${mark('carried')} | ${mark('partial')} | ${mark('dropped')} | ${travels} |`,
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
lines.push('');
|
|
241
|
+
|
|
242
|
+
lines.push('## Corpus declarations against the uncarried set');
|
|
243
|
+
lines.push('');
|
|
244
|
+
lines.push(
|
|
245
|
+
'How many skills declare each field the carrier drops or truncates. A field absent from',
|
|
246
|
+
);
|
|
247
|
+
lines.push('this table is declared by no skill today — the gap for it is theoretical.');
|
|
248
|
+
lines.push('');
|
|
249
|
+
const declared = c.fields
|
|
250
|
+
.filter((r) => r.column !== 'carried')
|
|
251
|
+
.map((r) => ({ field: r.field, column: r.column, n: c.perField[r.field] ?? 0 }))
|
|
252
|
+
.filter((r) => r.n > 0)
|
|
253
|
+
.sort((a, b) => b.n - a.n || a.field.localeCompare(b.field));
|
|
254
|
+
lines.push('| field | column | skills declaring |');
|
|
255
|
+
lines.push('|---|---|---|');
|
|
256
|
+
for (const r of declared) {
|
|
257
|
+
lines.push(`| \`${r.field}\` | ${r.column} | ${String(r.n)} |`);
|
|
258
|
+
}
|
|
259
|
+
lines.push('');
|
|
260
|
+
|
|
261
|
+
const nonPortable = c.skills.filter((s) => s.dropped.length > 0 || s.partial.length > 0);
|
|
262
|
+
lines.push(
|
|
263
|
+
`**${String(nonPortable.length)} of ${String(c.scanned)} skills declare at least one semantic the carrier does not transport whole.**`,
|
|
264
|
+
);
|
|
265
|
+
lines.push('');
|
|
266
|
+
|
|
267
|
+
lines.push('## Per skill');
|
|
268
|
+
lines.push('');
|
|
269
|
+
lines.push('| skill | dropped | partial |');
|
|
270
|
+
lines.push('|---|---|---|');
|
|
271
|
+
for (const s of c.skills) {
|
|
272
|
+
const d = s.dropped.length ? s.dropped.map((k) => `\`${k}\``).join(', ') : '—';
|
|
273
|
+
const pa = s.partial.length ? s.partial.map((k) => `\`${k}\``).join(', ') : '—';
|
|
274
|
+
lines.push(`| \`${s.skill}\` | ${d} | ${pa} |`);
|
|
275
|
+
}
|
|
276
|
+
lines.push('');
|
|
277
|
+
return lines.join('\n');
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
export function main(argv: readonly string[]): number {
|
|
281
|
+
const write = argv.includes('--write');
|
|
282
|
+
const json = argv.includes('--json');
|
|
283
|
+
const outIdx = argv.indexOf('--out');
|
|
284
|
+
const out = outIdx >= 0 ? argv[outIdx + 1] : undefined;
|
|
285
|
+
|
|
286
|
+
const c = census(REPO_ROOT);
|
|
287
|
+
|
|
288
|
+
if (json) {
|
|
289
|
+
process.stdout.write(`${JSON.stringify(c, null, 2)}\n`);
|
|
290
|
+
return 0;
|
|
291
|
+
}
|
|
292
|
+
if (write) {
|
|
293
|
+
const target = path.resolve(REPO_ROOT, out ?? DEFAULT_OUT);
|
|
294
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
295
|
+
fs.writeFileSync(target, render(c, pin(REPO_ROOT)), 'utf8');
|
|
296
|
+
process.stdout.write(
|
|
297
|
+
`semantic_parity_census: wrote ${path.relative(REPO_ROOT, target)}\n`,
|
|
298
|
+
);
|
|
299
|
+
return 0;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const counts: Record<Column, number> = { carried: 0, partial: 0, dropped: 0 };
|
|
303
|
+
for (const r of c.fields) counts[r.column] += 1;
|
|
304
|
+
process.stdout.write(
|
|
305
|
+
`semantic_parity_census · ${String(c.fields.length)} schema field(s): ` +
|
|
306
|
+
`${String(counts.carried)} carried, ${String(counts.partial)} partial, ${String(counts.dropped)} dropped\n`,
|
|
307
|
+
);
|
|
308
|
+
process.stdout.write(` corpus: ${String(c.scanned)} SKILL.md file(s)\n`);
|
|
309
|
+
return 0;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
|
|
313
|
+
process.exit(main(process.argv.slice(2)));
|
|
314
|
+
}
|
|
@@ -31,7 +31,15 @@ import process from 'node:process';
|
|
|
31
31
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
32
32
|
import * as path from 'node:path';
|
|
33
33
|
|
|
34
|
-
import {
|
|
34
|
+
import {
|
|
35
|
+
DEFAULT_PROJECTS_ROOT,
|
|
36
|
+
derivedBytes,
|
|
37
|
+
listTranscriptFiles,
|
|
38
|
+
measureBytesPerToken,
|
|
39
|
+
scanTranscripts,
|
|
40
|
+
type BytesPerToken,
|
|
41
|
+
type DerivedBytes,
|
|
42
|
+
} from './_lib/cc_transcript.js';
|
|
35
43
|
import { emptyCounters, scanEolSlice, type EolCounters } from './_lib/session_eol.js';
|
|
36
44
|
|
|
37
45
|
export interface SessionSummary {
|
|
@@ -90,6 +98,28 @@ export interface EolReport {
|
|
|
90
98
|
};
|
|
91
99
|
/** Pearson r of file bytes vs final context tokens over the readable set. */
|
|
92
100
|
bytes_tokens_pearson_r: number | null;
|
|
101
|
+
/**
|
|
102
|
+
* The median session's final context expressed in bytes - DERIVED, and
|
|
103
|
+
* labelled so (`road-to-a-bytes-row-that-exists` Phase 2, metric
|
|
104
|
+
* `provider-bytes-per-call`).
|
|
105
|
+
*
|
|
106
|
+
* Do NOT confuse this with `SessionSummary.bytes` or with the Pearson r
|
|
107
|
+
* above: those are the transcript FILE's size on this disk, a local
|
|
108
|
+
* artefact. This is an estimate of the context volume a round re-sends,
|
|
109
|
+
* obtained by multiplying a token count by a ratio measured on this same
|
|
110
|
+
* store. `null` whenever either input is missing - there is no fallback
|
|
111
|
+
* ratio anywhere in this path, deliberately.
|
|
112
|
+
*
|
|
113
|
+
* COST, named rather than discovered: producing it costs a second pass
|
|
114
|
+
* over the store (`scanTranscripts`), because the end-of-life counters are
|
|
115
|
+
* accumulated by `_lib/session_eol.ts`, whose shape is PERSISTED in state
|
|
116
|
+
* files and whose fields therefore carry a migration obligation this
|
|
117
|
+
* report does not need to take on for a derived column.
|
|
118
|
+
*/
|
|
119
|
+
provider_bytes: {
|
|
120
|
+
factor: BytesPerToken | null;
|
|
121
|
+
median_final_context: DerivedBytes | null;
|
|
122
|
+
};
|
|
93
123
|
threshold_crossings: Record<string, number>;
|
|
94
124
|
}
|
|
95
125
|
|
|
@@ -220,6 +250,11 @@ export function buildReport(root: string, now: Date = new Date()): EolReport {
|
|
|
220
250
|
const post = allEvents.map((e) => e.post_tokens).filter((v): v is number => v !== null);
|
|
221
251
|
const summaryRecords = sessions.reduce((s, x) => s + x.counters.compact_summaries, 0);
|
|
222
252
|
|
|
253
|
+
// Byte factor, measured on this same store. Separate pass - see the
|
|
254
|
+
// `provider_bytes` doc comment for why it is not folded into EolCounters.
|
|
255
|
+
const byteScan = scanTranscripts({ root, measureBytes: true });
|
|
256
|
+
const bytesFactor = measureBytesPerToken(byteScan.records);
|
|
257
|
+
|
|
223
258
|
const crossings: Record<string, number> = {};
|
|
224
259
|
for (const thr of [200_000, 400_000, 600_000, 800_000]) {
|
|
225
260
|
crossings[`>=${thr / 1000}k`] = finals.filter((c) => c >= thr).length;
|
|
@@ -250,6 +285,10 @@ export function buildReport(root: string, now: Date = new Date()): EolReport {
|
|
|
250
285
|
.filter((s) => (s.counters.final_context_tokens as number) > 1_000)
|
|
251
286
|
.map((s) => [s.bytes, s.counters.final_context_tokens as number]),
|
|
252
287
|
),
|
|
288
|
+
provider_bytes: {
|
|
289
|
+
factor: bytesFactor,
|
|
290
|
+
median_final_context: derivedBytes(percentiles(finals)?.median ?? 0, finals.length > 0 ? bytesFactor : null),
|
|
291
|
+
},
|
|
253
292
|
threshold_crossings: crossings,
|
|
254
293
|
};
|
|
255
294
|
}
|
|
@@ -274,6 +313,20 @@ export function renderText(r: EolReport): string {
|
|
|
274
313
|
if (r.turns) {
|
|
275
314
|
lines.push(`turns per session: median=${fmt(r.turns.median)} p90=${fmt(r.turns.p90)} max=${fmt(r.turns.max)}`);
|
|
276
315
|
}
|
|
316
|
+
const pb = r.provider_bytes;
|
|
317
|
+
if (pb.factor === null || pb.median_final_context === null) {
|
|
318
|
+
lines.push('provider bytes: basis unavailable - this store supplied no bytes-per-token factor');
|
|
319
|
+
} else {
|
|
320
|
+
lines.push(
|
|
321
|
+
`provider bytes (basis: ${pb.median_final_context.basis}): median final context`
|
|
322
|
+
+ ` ~= ${fmt(pb.median_final_context.bytes)} bytes`,
|
|
323
|
+
);
|
|
324
|
+
lines.push(
|
|
325
|
+
` factor ${pb.factor.bytes_per_token.toFixed(4)} bytes/token (basis: ${pb.factor.basis},`
|
|
326
|
+
+ ` ${fmt(pb.factor.content_bytes)} content bytes / ${fmt(pb.factor.output_tokens)} output tokens,`
|
|
327
|
+
+ ` n=${pb.factor.records}) - measured on assistant output, applied to an input token count`,
|
|
328
|
+
);
|
|
329
|
+
}
|
|
277
330
|
const c = r.compaction;
|
|
278
331
|
lines.push(
|
|
279
332
|
`compaction: ${c.sessions_with_compaction} sessions, ${c.events_total} events (${Object.entries(c.events_by_trigger)
|
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
"sequential-thinking"
|
|
6
6
|
],
|
|
7
7
|
"skills": [
|
|
8
|
-
"accessibility-auditor",
|
|
9
8
|
"activation-design",
|
|
10
9
|
"agent-docs-writing",
|
|
11
10
|
"agents-md-thin-root",
|
|
@@ -192,7 +191,6 @@
|
|
|
192
191
|
"throughput-vs-morale-tradeoff",
|
|
193
192
|
"token-optimizer",
|
|
194
193
|
"traefik",
|
|
195
|
-
"ui-component-architect",
|
|
196
194
|
"unit-economics-modeling",
|
|
197
195
|
"universal-project-analysis",
|
|
198
196
|
"upstream-contribute",
|
|
@@ -19,9 +19,11 @@
|
|
|
19
19
|
* - `--dry-run` (MockRouter, no key, no spend) exercises the plumbing only;
|
|
20
20
|
* floors are REPORTED but never fail the run — mock routing says nothing
|
|
21
21
|
* about real trigger accuracy.
|
|
22
|
-
* - Rotation is a pure function of (week index,
|
|
23
|
-
*
|
|
24
|
-
*
|
|
22
|
+
* - Rotation is a pure function of (week index, suite identity): a suite's
|
|
23
|
+
* slot comes from a hash of its own name, so every suite is visited once per
|
|
24
|
+
* `ROTATION_CYCLE_WEEKS` and adding one shifts nobody else. What floats is
|
|
25
|
+
* how many run in a given week; `rotation_plan` reports that and the weekly
|
|
26
|
+
* query cost it implies.
|
|
25
27
|
*
|
|
26
28
|
* Exit codes: 0 pass (always, in --dry-run) · 1 live floor breach ·
|
|
27
29
|
* 2 usage / IO error.
|
|
@@ -48,7 +50,6 @@ const _HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
|
48
50
|
const REPO_ROOT = path.resolve(_HERE, '..', '..');
|
|
49
51
|
const SKILLS_DIR = path.join(REPO_ROOT, 'src', 'skills');
|
|
50
52
|
const DEFAULT_OUT_DIR = path.join(REPO_ROOT, 'internal', 'evals', 'results', 'rotation');
|
|
51
|
-
const DEFAULT_BATCH = 5;
|
|
52
53
|
|
|
53
54
|
/** Router that may expose an async route (fetch-based live routers). */
|
|
54
55
|
type AsyncCapableRouter = TriggerRouter & {
|
|
@@ -64,23 +65,116 @@ export function week_index(d: Date): number {
|
|
|
64
65
|
}
|
|
65
66
|
|
|
66
67
|
/**
|
|
67
|
-
*
|
|
68
|
-
* `
|
|
69
|
-
* ceil(total / batch) weeks; adding/removing suites shifts but never
|
|
70
|
-
* starves the cycle.
|
|
68
|
+
* Weeks in one full rotation cycle. 12 × 7 = 84 days, inside the 90-day
|
|
69
|
+
* `MAX_AGE_DAYS` window `check_trigger_evals` enforces.
|
|
71
70
|
*/
|
|
72
|
-
export
|
|
73
|
-
|
|
71
|
+
export const ROTATION_CYCLE_WEEKS = 12;
|
|
72
|
+
|
|
73
|
+
/** Provider queries one suite's live pass costs, measured at ~9.5; rounded up. */
|
|
74
|
+
export const QUERIES_PER_SUITE = 10;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Stated weekly ceiling for the rotation's paid calls. Growth past it does not
|
|
78
|
+
* silently raise the bill — `rotation_plan` reports `withinCeiling: false` and
|
|
79
|
+
* a test asserts the live suite list stays under it, so crossing it is a red a
|
|
80
|
+
* human decides on rather than an invoice they discover.
|
|
81
|
+
*
|
|
82
|
+
* **Set from measurement, and the arithmetic is the point.** At 102 suites a
|
|
83
|
+
* full pass costs ~1020 queries; spread over a 12-week cycle that is ~85 a week
|
|
84
|
+
* on average and 140 in the busiest week (hash spread, measured 2026-09-30).
|
|
85
|
+
* 160 leaves headroom for that unevenness and reds at roughly 117 suites.
|
|
86
|
+
*
|
|
87
|
+
* The number nobody can lower by tuning: a 90-day window over N suites costs
|
|
88
|
+
* `N × QUERIES_PER_SUITE` every 90 days, whatever the schedule. The scheme this
|
|
89
|
+
* replaced looked cheaper only because it never completed a pass — `batch = 5`
|
|
90
|
+
* over 102 suites is a 21-week cycle, which misses the 90-day window by 57 days.
|
|
91
|
+
* Cheapness was the symptom of the defect, not a property worth preserving.
|
|
92
|
+
*/
|
|
93
|
+
export const MAX_WEEKLY_QUERIES = 160;
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* FNV-1a over the suite id. Any stable hash works; what matters is that a
|
|
97
|
+
* suite's slot depends on the suite alone and on nothing about the list it
|
|
98
|
+
* sits in.
|
|
99
|
+
*/
|
|
100
|
+
function slot_of(id: string, cycleWeeks: number): number {
|
|
101
|
+
let h = 0x811c9dc5;
|
|
102
|
+
for (let i = 0; i < id.length; i += 1) {
|
|
103
|
+
h ^= id.charCodeAt(i);
|
|
104
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
105
|
+
}
|
|
106
|
+
return h % cycleWeeks;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Deterministic rotation by suite identity: a suite is due in the weeks whose
|
|
111
|
+
* `week mod cycleWeeks` equals its own slot, so worst-case staleness is exactly
|
|
112
|
+
* `cycleWeeks` for any suite present across a cycle — independent of how many
|
|
113
|
+
* suites exist.
|
|
114
|
+
*
|
|
115
|
+
* **Why not the positional window this replaces.** It selected `batch`
|
|
116
|
+
* consecutive suites from `start = (week * batch) mod total`, and that start
|
|
117
|
+
* re-bases whenever `total` changes, so a region can be skipped repeatedly.
|
|
118
|
+
* Simulated against a list growing by one suite every eight weeks over five
|
|
119
|
+
* years, worst-case staleness reached **24 weeks against a 12-week window**;
|
|
120
|
+
* `trigger_eval_rotation_growth.test.ts` is that simulation, and it was
|
|
121
|
+
* observed red against the old scheme before this replaced it. A loop over
|
|
122
|
+
* static totals cannot see the defect, which is why the earlier test suite
|
|
123
|
+
* passed while the property was false.
|
|
124
|
+
*
|
|
125
|
+
* The cost of the fix is that the batch is no longer a knob: how many suites
|
|
126
|
+
* run in a given week follows from the hash distribution, and `rotation_plan`
|
|
127
|
+
* reports it rather than leaving it to be discovered on an invoice.
|
|
128
|
+
*/
|
|
129
|
+
export function pick_rotation<T>(
|
|
130
|
+
suites: readonly T[],
|
|
131
|
+
week: number,
|
|
132
|
+
cycleWeeks: number = ROTATION_CYCLE_WEEKS,
|
|
133
|
+
): T[] {
|
|
134
|
+
if (suites.length === 0 || cycleWeeks <= 0) {
|
|
74
135
|
return [];
|
|
75
136
|
}
|
|
76
|
-
const
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
137
|
+
const due = (((week % cycleWeeks) + cycleWeeks) % cycleWeeks);
|
|
138
|
+
return suites.filter((s) => slot_of(String(s), cycleWeeks) === due);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** What a given suite list costs and promises, derivable without running it. */
|
|
142
|
+
export interface RotationPlan {
|
|
143
|
+
total: number;
|
|
144
|
+
cycleWeeks: number;
|
|
145
|
+
/** Suites in the busiest week of the cycle. */
|
|
146
|
+
peakSuitesPerWeek: number;
|
|
147
|
+
/** Provider queries in that busiest week. */
|
|
148
|
+
peakWeeklyQueries: number;
|
|
149
|
+
/** Worst-case days between two visits of one suite. */
|
|
150
|
+
worstCaseStalenessDays: number;
|
|
151
|
+
withinCeiling: boolean;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Derive the cost and staleness a suite list implies. Pure, so the weekly bill
|
|
156
|
+
* is readable from the code rather than from a run.
|
|
157
|
+
*/
|
|
158
|
+
export function rotation_plan(
|
|
159
|
+
suites: readonly string[],
|
|
160
|
+
cycleWeeks: number = ROTATION_CYCLE_WEEKS,
|
|
161
|
+
): RotationPlan {
|
|
162
|
+
const span = Math.max(cycleWeeks, 1);
|
|
163
|
+
const counts = new Array<number>(span).fill(0);
|
|
164
|
+
for (const s of suites) {
|
|
165
|
+
const slot = slot_of(s, span);
|
|
166
|
+
counts[slot] = (counts[slot] ?? 0) + 1;
|
|
82
167
|
}
|
|
83
|
-
|
|
168
|
+
const peakSuitesPerWeek = counts.length > 0 ? Math.max(...counts) : 0;
|
|
169
|
+
const peakWeeklyQueries = peakSuitesPerWeek * QUERIES_PER_SUITE;
|
|
170
|
+
return {
|
|
171
|
+
total: suites.length,
|
|
172
|
+
cycleWeeks,
|
|
173
|
+
peakSuitesPerWeek,
|
|
174
|
+
peakWeeklyQueries,
|
|
175
|
+
worstCaseStalenessDays: cycleWeeks * 7,
|
|
176
|
+
withinCeiling: peakWeeklyQueries <= MAX_WEEKLY_QUERIES,
|
|
177
|
+
};
|
|
84
178
|
}
|
|
85
179
|
|
|
86
180
|
/** Enumerate skills carrying `evals/triggers.json`, sorted for determinism. */
|
|
@@ -181,7 +275,8 @@ export async function run_eval_async(
|
|
|
181
275
|
|
|
182
276
|
export interface RotationOptions {
|
|
183
277
|
week?: number;
|
|
184
|
-
|
|
278
|
+
/** Weeks in a full cycle; also the worst-case staleness. Default 12. */
|
|
279
|
+
cycleWeeks?: number;
|
|
185
280
|
dryRun?: boolean;
|
|
186
281
|
outDir?: string;
|
|
187
282
|
model?: string;
|
|
@@ -202,7 +297,7 @@ export interface RotationSuiteOutcome {
|
|
|
202
297
|
|
|
203
298
|
export interface RotationSummary {
|
|
204
299
|
week: number;
|
|
205
|
-
|
|
300
|
+
cycle_weeks: number;
|
|
206
301
|
total_suites: number;
|
|
207
302
|
dry_run: boolean;
|
|
208
303
|
outcomes: RotationSuiteOutcome[];
|
|
@@ -211,14 +306,14 @@ export interface RotationSummary {
|
|
|
211
306
|
|
|
212
307
|
export async function run_rotation(opts: RotationOptions = {}): Promise<RotationSummary> {
|
|
213
308
|
const dryRun = opts.dryRun ?? false;
|
|
214
|
-
const
|
|
309
|
+
const cycleWeeks = opts.cycleWeeks ?? ROTATION_CYCLE_WEEKS;
|
|
215
310
|
const week = opts.week ?? week_index(new Date());
|
|
216
311
|
const outDir = opts.outDir ?? DEFAULT_OUT_DIR;
|
|
217
312
|
const model = opts.model ?? DEFAULT_MODEL;
|
|
218
313
|
const skillsDir = opts.skillsDir ?? SKILLS_DIR;
|
|
219
314
|
|
|
220
315
|
const suites = list_trigger_suites(skillsDir);
|
|
221
|
-
const picked = pick_rotation(suites, week,
|
|
316
|
+
const picked = pick_rotation(suites, week, cycleWeeks);
|
|
222
317
|
const catalogue = load_skill_metas();
|
|
223
318
|
|
|
224
319
|
const router: AsyncCapableRouter =
|
|
@@ -257,7 +352,7 @@ export async function run_rotation(opts: RotationOptions = {}): Promise<Rotation
|
|
|
257
352
|
|
|
258
353
|
return {
|
|
259
354
|
week,
|
|
260
|
-
|
|
355
|
+
cycle_weeks: cycleWeeks,
|
|
261
356
|
total_suites: suites.length,
|
|
262
357
|
dry_run: dryRun,
|
|
263
358
|
outcomes,
|
|
@@ -278,7 +373,7 @@ function parse_args(argv: string[]): RotationOptions & { help?: boolean } {
|
|
|
278
373
|
return v as string;
|
|
279
374
|
};
|
|
280
375
|
if (a === '--week') out.week = Number(take());
|
|
281
|
-
else if (a === '--
|
|
376
|
+
else if (a === '--cycle-weeks') out.cycleWeeks = Number(take());
|
|
282
377
|
else if (a === '--dry-run') out.dryRun = true;
|
|
283
378
|
else if (a === '--out-dir') out.outDir = path.resolve(take());
|
|
284
379
|
else if (a === '--model') out.model = take();
|
|
@@ -292,8 +387,8 @@ function parse_args(argv: string[]): RotationOptions & { help?: boolean } {
|
|
|
292
387
|
process.stderr.write('trigger_eval_rotation: error: --week must be an integer\n');
|
|
293
388
|
process.exit(2);
|
|
294
389
|
}
|
|
295
|
-
if (out.
|
|
296
|
-
process.stderr.write('trigger_eval_rotation: error: --
|
|
390
|
+
if (out.cycleWeeks !== undefined && (!Number.isInteger(out.cycleWeeks) || out.cycleWeeks <= 0)) {
|
|
391
|
+
process.stderr.write('trigger_eval_rotation: error: --cycle-weeks must be a positive integer\n');
|
|
297
392
|
process.exit(2);
|
|
298
393
|
}
|
|
299
394
|
return out;
|
|
@@ -303,7 +398,7 @@ export async function main(argv?: string[]): Promise<number> {
|
|
|
303
398
|
const opts = parse_args(argv ?? process.argv.slice(2));
|
|
304
399
|
if (opts.help) {
|
|
305
400
|
process.stdout.write(
|
|
306
|
-
'usage: trigger_eval_rotation [--week N] [--
|
|
401
|
+
'usage: trigger_eval_rotation [--week N] [--cycle-weeks N] [--dry-run]\n' +
|
|
307
402
|
' [--out-dir DIR] [--model MODEL]\n',
|
|
308
403
|
);
|
|
309
404
|
return 0;
|
|
@@ -316,7 +411,7 @@ export async function main(argv?: string[]): Promise<number> {
|
|
|
316
411
|
return 2;
|
|
317
412
|
}
|
|
318
413
|
process.stdout.write(
|
|
319
|
-
`trigger-eval rotation · week=${summary.week} ·
|
|
414
|
+
`trigger-eval rotation · week=${summary.week} · cycle=${summary.cycle_weeks}w · ` +
|
|
320
415
|
`${summary.outcomes.length}/${summary.total_suites} suites this run` +
|
|
321
416
|
`${summary.dry_run ? ' · DRY-RUN (floors advisory)' : ''}\n`,
|
|
322
417
|
);
|