openmerit 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/README.md +83 -372
  3. package/dist/core/src/index.d.ts +90 -0
  4. package/dist/core/src/index.js +1137 -0
  5. package/dist/core/src/store.d.ts +35 -0
  6. package/dist/core/src/store.js +102 -0
  7. package/dist/pi/src/index.d.ts +15 -0
  8. package/dist/pi/src/index.js +423 -0
  9. package/dist/protocol/src/index.d.ts +402 -0
  10. package/dist/protocol/src/index.js +47 -0
  11. package/dist/protocol/src/schemas.d.ts +450 -0
  12. package/dist/protocol/src/schemas.js +224 -0
  13. package/docs/adapter-guide.md +172 -0
  14. package/docs/architecture.md +55 -0
  15. package/docs/automation.md +66 -0
  16. package/docs/getting-started.md +55 -0
  17. package/docs/lifecycle.md +30 -0
  18. package/docs/metrics-and-evidence.md +40 -0
  19. package/docs/operations.md +31 -0
  20. package/docs/pareto-spec.md +76 -0
  21. package/docs/pi-extension.md +44 -0
  22. package/docs/roadmap.md +26 -0
  23. package/docs/security.md +23 -0
  24. package/docs/testing.md +36 -0
  25. package/docs/ux-reference.md +32 -0
  26. package/package.json +45 -54
  27. package/benchmark/invoice_ocr/data/invoice_01_ground_truth.json +0 -38
  28. package/benchmark/invoice_ocr/data/invoice_01_row_2.jpg +0 -0
  29. package/benchmark/invoice_ocr/data/invoice_02_ground_truth.json +0 -32
  30. package/benchmark/invoice_ocr/data/invoice_02_row_5.jpg +0 -0
  31. package/benchmark/invoice_ocr/data/invoice_03_ground_truth.json +0 -26
  32. package/benchmark/invoice_ocr/data/invoice_03_row_6.jpg +0 -0
  33. package/benchmark/invoice_ocr/data/invoice_04_ground_truth.json +0 -26
  34. package/benchmark/invoice_ocr/data/invoice_04_row_7.jpg +0 -0
  35. package/benchmark/invoice_ocr/data/invoice_05_ground_truth.json +0 -38
  36. package/benchmark/invoice_ocr/data/invoice_05_row_947.jpg +0 -0
  37. package/benchmark/invoice_ocr/data/invoice_06_ground_truth.json +0 -38
  38. package/benchmark/invoice_ocr/data/invoice_06_row_948.jpg +0 -0
  39. package/benchmark/invoice_ocr/data/invoice_07_ground_truth.json +0 -20
  40. package/benchmark/invoice_ocr/data/invoice_07_row_949.jpg +0 -0
  41. package/benchmark/invoice_ocr/data/invoice_08_ground_truth.json +0 -38
  42. package/benchmark/invoice_ocr/data/invoice_08_row_1888.jpg +0 -0
  43. package/benchmark/invoice_ocr/data/invoice_09_ground_truth.json +0 -26
  44. package/benchmark/invoice_ocr/data/invoice_09_row_1890.jpg +0 -0
  45. package/benchmark/invoice_ocr/data/invoice_10_ground_truth.json +0 -20
  46. package/benchmark/invoice_ocr/data/invoice_10_row_1892.jpg +0 -0
  47. package/benchmark/invoice_ocr/data/manifest.json +0 -97
  48. package/dist/benchmarks.js +0 -98
  49. package/dist/catalog.js +0 -61
  50. package/dist/cli.js +0 -188
  51. package/dist/daemon.js +0 -407
  52. package/dist/diagnostics.js +0 -227
  53. package/dist/frontier.js +0 -56
  54. package/dist/harness.js +0 -1
  55. package/dist/integrations.js +0 -19
  56. package/dist/invoice-eval.js +0 -33
  57. package/dist/invoice-score.js +0 -124
  58. package/dist/judge.js +0 -43
  59. package/dist/llm.js +0 -207
  60. package/dist/pi-config.js +0 -46
  61. package/dist/pi-trials.js +0 -373
  62. package/dist/policy.js +0 -185
  63. package/dist/providers.js +0 -1
  64. package/dist/recommend.js +0 -76
  65. package/dist/routes.js +0 -74
  66. package/dist/standalone.js +0 -224
  67. package/dist/store.js +0 -89
  68. package/dist/strategist.js +0 -68
  69. package/dist/task-input.js +0 -54
  70. package/dist/traces.js +0 -127
  71. package/dist/trials.js +0 -140
  72. package/dist/types.js +0 -2
  73. package/examples/invoice-prompt.txt +0 -19
  74. package/examples/task.example.json +0 -7
  75. package/extension/openmerit.ts +0 -947
  76. package/instructions/OPENMERIT.md +0 -63
  77. package/instructions/openmerit.policy.json +0 -37
  78. package/rules.md +0 -43
@@ -0,0 +1,35 @@
1
+ import type { EvidenceManifest, OpenMeritAuditEvent, OpenMeritProjectConfig, OpenMeritProjectState } from "../../protocol/src/index.ts";
2
+ export declare const OPENMERIT_DIRECTORY = ".openmerit";
3
+ export interface OpenMeritStore {
4
+ initialize(config: OpenMeritProjectConfig, state: OpenMeritProjectState): Promise<void>;
5
+ readConfig(): Promise<OpenMeritProjectConfig | undefined>;
6
+ writeConfig(config: OpenMeritProjectConfig): Promise<void>;
7
+ readState(): Promise<OpenMeritProjectState | undefined>;
8
+ writeState(state: OpenMeritProjectState): Promise<void>;
9
+ appendEvent(event: OpenMeritAuditEvent): Promise<void>;
10
+ writeEvidenceManifest(manifest: EvidenceManifest): Promise<string>;
11
+ }
12
+ export interface AuditEventExporter {
13
+ readonly id: string;
14
+ export(event: OpenMeritAuditEvent): Promise<void>;
15
+ }
16
+ export interface ProjectStoreOptions {
17
+ readonly eventExporters?: readonly AuditEventExporter[];
18
+ }
19
+ export declare class ProjectStore implements OpenMeritStore {
20
+ readonly directory: string;
21
+ readonly configPath: string;
22
+ readonly statePath: string;
23
+ readonly eventsPath: string;
24
+ readonly exportErrorsPath: string;
25
+ readonly evidenceDirectory: string;
26
+ private readonly eventExporters;
27
+ constructor(projectRoot: string, options?: ProjectStoreOptions);
28
+ initialize(config: OpenMeritProjectConfig, state: OpenMeritProjectState): Promise<void>;
29
+ readConfig(): Promise<OpenMeritProjectConfig | undefined>;
30
+ writeConfig(config: OpenMeritProjectConfig): Promise<void>;
31
+ readState(): Promise<OpenMeritProjectState | undefined>;
32
+ writeState(state: OpenMeritProjectState): Promise<void>;
33
+ appendEvent(event: OpenMeritAuditEvent): Promise<void>;
34
+ writeEvidenceManifest(manifest: EvidenceManifest): Promise<string>;
35
+ }
@@ -0,0 +1,102 @@
1
+ import { appendFile, mkdir, readFile, rename, writeFile } from "node:fs/promises";
2
+ import { dirname, join } from "node:path";
3
+ export const OPENMERIT_DIRECTORY = ".openmerit";
4
+ export class ProjectStore {
5
+ directory;
6
+ configPath;
7
+ statePath;
8
+ eventsPath;
9
+ exportErrorsPath;
10
+ evidenceDirectory;
11
+ eventExporters;
12
+ constructor(projectRoot, options = {}) {
13
+ this.directory = join(projectRoot, OPENMERIT_DIRECTORY);
14
+ this.configPath = join(this.directory, "config.json");
15
+ this.statePath = join(this.directory, "state.json");
16
+ this.eventsPath = join(this.directory, "events.jsonl");
17
+ this.exportErrorsPath = join(this.directory, "audit-export-errors.jsonl");
18
+ this.evidenceDirectory = join(this.directory, "evidence");
19
+ this.eventExporters = options.eventExporters ?? [];
20
+ }
21
+ async initialize(config, state) {
22
+ await mkdir(this.evidenceDirectory, { recursive: true });
23
+ await this.writeConfig(config);
24
+ await this.writeState(state);
25
+ }
26
+ async readConfig() {
27
+ return readJsonIfPresent(this.configPath);
28
+ }
29
+ async writeConfig(config) {
30
+ await atomicWriteJson(this.configPath, config);
31
+ }
32
+ async readState() {
33
+ return readJsonIfPresent(this.statePath);
34
+ }
35
+ async writeState(state) {
36
+ await atomicWriteJson(this.statePath, state);
37
+ }
38
+ async appendEvent(event) {
39
+ await mkdir(dirname(this.eventsPath), { recursive: true });
40
+ const safeEvent = redactSensitive(event);
41
+ await appendFile(this.eventsPath, `${JSON.stringify(safeEvent)}\n`, { encoding: "utf8", mode: 0o600 });
42
+ const results = await Promise.allSettled(this.eventExporters.map((exporter) => exporter.export(safeEvent)));
43
+ for (const [index, result] of results.entries()) {
44
+ if (result.status === "fulfilled")
45
+ continue;
46
+ await appendFile(this.exportErrorsPath, `${JSON.stringify({
47
+ occurredAt: new Date().toISOString(),
48
+ eventId: safeEvent.id,
49
+ exporterId: this.eventExporters[index]?.id ?? "unknown",
50
+ error: safeErrorMessage(result.reason),
51
+ })}\n`, { encoding: "utf8", mode: 0o600 });
52
+ }
53
+ }
54
+ async writeEvidenceManifest(manifest) {
55
+ const path = join(this.evidenceDirectory, `${safeFileComponent(manifest.id)}.json`);
56
+ await atomicWriteJson(path, redactSensitive(manifest));
57
+ return path;
58
+ }
59
+ }
60
+ const SENSITIVE_KEY = /(?:authorization|api[_-]?key|token|secret|password|cookie)/i;
61
+ const SECRET_VALUE = /\b(?:sk-[a-zA-Z0-9_-]{12,}|apikey_[a-zA-Z0-9_-]{12,})\b/g;
62
+ function redactSensitive(value, key) {
63
+ if (key && SENSITIVE_KEY.test(key))
64
+ return "[REDACTED]";
65
+ if (typeof value === "string")
66
+ return value.replaceAll(SECRET_VALUE, "[REDACTED]");
67
+ if (Array.isArray(value))
68
+ return value.map((item) => redactSensitive(item));
69
+ if (value && typeof value === "object") {
70
+ return Object.fromEntries(Object.entries(value).map(([childKey, child]) => [
71
+ childKey,
72
+ redactSensitive(child, childKey),
73
+ ]));
74
+ }
75
+ return value;
76
+ }
77
+ function safeErrorMessage(reason) {
78
+ const message = reason instanceof Error ? reason.message : String(reason);
79
+ return redactSensitive(message);
80
+ }
81
+ function safeFileComponent(value) {
82
+ const safe = value.replaceAll(/[^a-zA-Z0-9._-]/g, "-");
83
+ if (!safe || safe === "." || safe === "..")
84
+ throw new Error("Evidence manifest ID is not file-safe.");
85
+ return safe;
86
+ }
87
+ async function readJsonIfPresent(path) {
88
+ try {
89
+ return JSON.parse(await readFile(path, "utf8"));
90
+ }
91
+ catch (error) {
92
+ if (error.code === "ENOENT")
93
+ return undefined;
94
+ throw error;
95
+ }
96
+ }
97
+ async function atomicWriteJson(path, value) {
98
+ await mkdir(dirname(path), { recursive: true });
99
+ const temporaryPath = `${path}.${process.pid}.${crypto.randomUUID()}.tmp`;
100
+ await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
101
+ await rename(temporaryPath, path);
102
+ }
@@ -0,0 +1,15 @@
1
+ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
2
+ import { type HarnessDescriptor, type ImprovementStage } from "../../protocol/src/index.ts";
3
+ export declare const PI_HARNESS_DESCRIPTOR: HarnessDescriptor;
4
+ export interface OpenMeritUiState {
5
+ proactiveNudgesPaused: boolean;
6
+ taskProfile: "not_configured" | "pending" | "configured";
7
+ metricReadiness: "not_assessed" | "pending" | "assessed";
8
+ evaluationReadiness: "not_assessed" | "pending" | "assessed";
9
+ activeIntent: string | null;
10
+ nextAssessment: string | null;
11
+ lifecycleStage: ImprovementStage | "not_initialized";
12
+ }
13
+ export declare function createInitialUiState(): OpenMeritUiState;
14
+ export declare function renderStatus(state: OpenMeritUiState): string;
15
+ export default function openMeritExtension(pi: ExtensionAPI): void;
@@ -0,0 +1,423 @@
1
+ import { createHash } from "node:crypto";
2
+ import { StringEnum } from "@earendil-works/pi-ai";
3
+ import { nextImprovementAction, OpenMeritCoordinator, ProjectStore, } from "../../core/src/index.js";
4
+ import { completionToolSchemaFor, OPENMERIT_SCHEMA_DIALECT, PROTOCOL_VERSION, MetricRecordSchema, } from "../../protocol/src/index.js";
5
+ import { Type } from "typebox";
6
+ const INTENT_ENTRY_TYPE = "openmerit.intent";
7
+ const RESULT_ENTRY_TYPE = "openmerit.result";
8
+ const INTENT_MESSAGE_TYPE = "openmerit.intent-context";
9
+ export const PI_HARNESS_DESCRIPTOR = {
10
+ id: "pi",
11
+ name: "Pi coding agent",
12
+ version: "0.87",
13
+ protocolVersions: [PROTOCOL_VERSION],
14
+ capabilities: [
15
+ "user_confirmation",
16
+ "evaluation_authoring",
17
+ "observability_instrumentation",
18
+ "production_observation",
19
+ "candidate_discovery",
20
+ "challenger_execution",
21
+ "frontier_calculation",
22
+ "model_mutation",
23
+ "post_swap_verification",
24
+ "rollback",
25
+ ],
26
+ executionModes: ["interactive", "non_interactive"],
27
+ structuredOutput: {
28
+ schemaDialect: OPENMERIT_SCHEMA_DIALECT,
29
+ presentation: "dynamic_tool",
30
+ enforcement: "harness",
31
+ },
32
+ automation: {
33
+ supportedSignals: [
34
+ "product_task_completed", "metric_window_available", "scheduled_tick",
35
+ "model_catalog_changed", "verification_window_completed",
36
+ ],
37
+ persistentScheduling: false,
38
+ backgroundExecution: false,
39
+ wakeupProvisioning: "none",
40
+ },
41
+ };
42
+ export function createInitialUiState() {
43
+ return {
44
+ proactiveNudgesPaused: false,
45
+ taskProfile: "not_configured",
46
+ metricReadiness: "not_assessed",
47
+ evaluationReadiness: "not_assessed",
48
+ activeIntent: null,
49
+ nextAssessment: null,
50
+ lifecycleStage: "not_initialized",
51
+ };
52
+ }
53
+ export function renderStatus(state) {
54
+ return [
55
+ "OpenMerit evidence readiness",
56
+ `task profile : ${state.taskProfile.replaceAll("_", " ")}`,
57
+ `metric readiness : ${state.metricReadiness.replaceAll("_", " ")}`,
58
+ `evaluation status : ${state.evaluationReadiness.replaceAll("_", " ")}`,
59
+ `active work : ${state.activeIntent ?? "none"}`,
60
+ `next assessment : ${state.nextAssessment ?? "not scheduled"}`,
61
+ `frontier lifecycle : ${state.lifecycleStage.replaceAll("_", " ")}`,
62
+ `proactive nudges : ${state.proactiveNudgesPaused ? "paused" : "enabled"}`,
63
+ ].join("\n");
64
+ }
65
+ function modelCatalogFingerprint(ctx) {
66
+ const catalogue = ctx.modelRegistry.getAvailable().map((model) => ({
67
+ provider: model.provider,
68
+ id: model.id,
69
+ reasoning: model.reasoning,
70
+ input: model.input,
71
+ cost: model.cost,
72
+ contextWindow: model.contextWindow,
73
+ maxTokens: model.maxTokens,
74
+ })).sort((left, right) => `${left.provider}/${left.id}`.localeCompare(`${right.provider}/${right.id}`));
75
+ return createHash("sha256").update(JSON.stringify(catalogue)).digest("hex");
76
+ }
77
+ function privateIntentMessage(intent) {
78
+ return [
79
+ "OpenMerit has issued the typed intent below.",
80
+ "Perform the intelligent work using the current project, session context, and available tools.",
81
+ "Use harness-configured fast decision resources such as Jev System One aggressively for bounded ranking, triage, and judgment when available; OpenMerit does not call those services itself.",
82
+ "You, the coding harness, own all research, evaluation design, instrumentation, measurement, Pareto calculation, and authorized configuration work. OpenMerit only validates the returned contract and evidence.",
83
+ "Instrument every required metric selected by the user. Report explicit covered and missing metric IDs; never describe observability as ready while a required metric is missing.",
84
+ intent.constraints.modelMutationAllowed
85
+ ? "This intent authorizes only the model mutation described by the active OpenMerit cycle."
86
+ : "Do not change Pi's model or application model configuration.",
87
+ "Before finishing, call openmerit_complete_intent exactly once with the result and durable evidence references.",
88
+ JSON.stringify(intent),
89
+ ].join("\n\n");
90
+ }
91
+ function piHarnessAdapter(pi) {
92
+ return {
93
+ descriptor: PI_HARNESS_DESCRIPTOR,
94
+ async dispatch(intent) {
95
+ pi.appendEntry(INTENT_ENTRY_TYPE, { intent });
96
+ pi.sendMessage({
97
+ customType: INTENT_MESSAGE_TYPE,
98
+ content: privateIntentMessage(intent),
99
+ display: false,
100
+ details: { intentId: intent.id, kind: intent.kind },
101
+ }, { deliverAs: "followUp", triggerTurn: true });
102
+ return { accepted: true, harnessJobId: intent.id };
103
+ },
104
+ };
105
+ }
106
+ function restoreState(ctx, state) {
107
+ state.activeIntent = null;
108
+ for (const entry of ctx.sessionManager.getEntries()) {
109
+ if (entry.type !== "custom")
110
+ continue;
111
+ if (entry.customType === INTENT_ENTRY_TYPE) {
112
+ const stored = entry.data;
113
+ if (stored?.intent?.id)
114
+ state.activeIntent = stored.intent.id;
115
+ }
116
+ if (entry.customType === RESULT_ENTRY_TYPE) {
117
+ const stored = entry.data;
118
+ if (stored?.result?.intentId === state.activeIntent)
119
+ state.activeIntent = null;
120
+ }
121
+ }
122
+ }
123
+ async function dispatchIntent(pi, ctx, state, store, kind, prepareCompletionTool) {
124
+ if (state.activeIntent) {
125
+ ctx.ui.notify(`openmerit: work is already active (${state.activeIntent})`, "warning");
126
+ return;
127
+ }
128
+ prepareCompletionTool(kind);
129
+ const issued = await new OpenMeritCoordinator(store, piHarnessAdapter(pi)).issue(kind);
130
+ const { intent } = issued;
131
+ state.activeIntent = intent.id;
132
+ if (kind === "establish_evals") {
133
+ state.taskProfile = "pending";
134
+ state.metricReadiness = "pending";
135
+ state.evaluationReadiness = "pending";
136
+ }
137
+ else {
138
+ state.metricReadiness = "pending";
139
+ }
140
+ state.lifecycleStage = issued.state.cycle.stage;
141
+ const label = kind === "establish_evals" ? "setup" : kind.replaceAll("_", " ");
142
+ ctx.ui.notify(`openmerit: ${label} started (${intent.id})`, "info");
143
+ }
144
+ async function reportAutomationSignal(pi, ctx, state, store, signal, prepareCompletionTool) {
145
+ const result = await new OpenMeritCoordinator(store, piHarnessAdapter(pi)).recordAutomationSignal(signal);
146
+ if (result.decision.action === "issue_intent" && !state.activeIntent) {
147
+ await dispatchIntent(pi, ctx, state, store, result.decision.intentKind, prepareCompletionTool);
148
+ }
149
+ }
150
+ export default function openMeritExtension(pi) {
151
+ const state = createInitialUiState();
152
+ let store;
153
+ let suppressNextSettledObservation = false;
154
+ const registerCompletionTool = (kind) => {
155
+ const parameters = kind
156
+ ? completionToolSchemaFor(kind)
157
+ : Type.Object({
158
+ intentId: Type.String(),
159
+ status: StringEnum(["failed", "cancelled"]),
160
+ summary: Type.String(),
161
+ evidence: Type.Array(Type.Object({
162
+ id: Type.String(),
163
+ source: StringEnum([
164
+ "harness_trace", "evaluation_artifact", "observability_record", "external_source", "user_feedback",
165
+ ]),
166
+ uri: Type.String(),
167
+ mediaType: Type.Optional(Type.String()),
168
+ digest: Type.Optional(Type.String()),
169
+ })),
170
+ errorCode: Type.Optional(Type.String()),
171
+ errorMessage: Type.Optional(Type.String()),
172
+ retryable: Type.Optional(Type.Boolean()),
173
+ }, { additionalProperties: false });
174
+ pi.registerTool({
175
+ name: "openmerit_complete_intent",
176
+ label: "Complete OpenMerit Intent",
177
+ description: kind
178
+ ? `Complete the active ${kind} OpenMerit intent with its required structured output and durable evidence references.`
179
+ : "Complete the active OpenMerit intent. A successful result is available only after an intent-specific schema is activated.",
180
+ promptSnippet: "Complete the active OpenMerit intent with structured evidence",
181
+ promptGuidelines: [
182
+ "Call openmerit_complete_intent exactly once after completing OpenMerit-requested work.",
183
+ "Never use openmerit_complete_intent to claim success without verified evidence references.",
184
+ "A succeeded result must include intent-specific outputs and at least one evidence reference; failed or cancelled results may omit outputs and use an empty evidence list.",
185
+ ],
186
+ parameters,
187
+ constrainedSampling: { type: "json_schema", strict: "prefer" },
188
+ async execute(_toolCallId, input, _signal, _onUpdate, ctx) {
189
+ if (!state.activeIntent)
190
+ throw new Error("No OpenMerit intent is active.");
191
+ if (input.intentId !== state.activeIntent) {
192
+ throw new Error(`Intent ${input.intentId} is not active.`);
193
+ }
194
+ if (input.status === "succeeded" && input.evidence.length === 0) {
195
+ throw new Error("A successful OpenMerit intent requires at least one evidence reference.");
196
+ }
197
+ store ??= new ProjectStore(ctx.cwd);
198
+ const outputs = input.outputs ?? {};
199
+ const result = {
200
+ protocolVersion: PROTOCOL_VERSION,
201
+ intentId: input.intentId,
202
+ status: input.status,
203
+ completedAt: new Date().toISOString(),
204
+ summary: input.summary,
205
+ evidence: input.evidence,
206
+ outputs,
207
+ ...(input.status === "failed"
208
+ ? {
209
+ error: {
210
+ code: input.errorCode ?? "harness_work_failed",
211
+ message: input.errorMessage ?? input.summary,
212
+ retryable: input.retryable ?? false,
213
+ },
214
+ }
215
+ : {}),
216
+ };
217
+ const accepted = await new OpenMeritCoordinator(store, piHarnessAdapter(pi)).acceptResult(result);
218
+ pi.appendEntry(RESULT_ENTRY_TYPE, { result });
219
+ suppressNextSettledObservation = true;
220
+ const { intentKind, state: nextState } = accepted;
221
+ const stage = nextState.cycle.stage;
222
+ state.activeIntent = null;
223
+ state.lifecycleStage = stage;
224
+ registerCompletionTool();
225
+ state.nextAssessment = accepted.automation.plan.nextWakeupAt ??
226
+ (accepted.automation.plan.scheduling === "external_scheduler_required" ? "external scheduler required" : "signal driven");
227
+ if (accepted.automation.plan.gaps.length) {
228
+ ctx.ui.notify(`openmerit automation: ${accepted.automation.plan.gaps.join(" ")}`, "warning");
229
+ }
230
+ if (input.status === "succeeded" && intentKind === "establish_evals") {
231
+ state.taskProfile = "configured";
232
+ state.metricReadiness = nextState.observability?.status === "ready" ? "assessed" : "pending";
233
+ state.evaluationReadiness = "assessed";
234
+ }
235
+ else if (input.status === "succeeded" && intentKind === "instrument_observability") {
236
+ state.metricReadiness = nextState.observability?.status === "ready" ? "assessed" : "pending";
237
+ }
238
+ if (input.status === "succeeded" && !state.proactiveNudgesPaused) {
239
+ const decision = nextImprovementAction(nextState.cycle);
240
+ if (decision.action === "issue_intent") {
241
+ await dispatchIntent(pi, ctx, state, store, decision.intentKind, registerCompletionTool);
242
+ }
243
+ else if (decision.action === "await_user") {
244
+ ctx.ui.notify(`openmerit: ${decision.reason}`, "warning");
245
+ }
246
+ }
247
+ return {
248
+ content: [{ type: "text", text: `Recorded OpenMerit intent ${input.intentId} as ${input.status}.` }],
249
+ details: { result },
250
+ };
251
+ },
252
+ });
253
+ };
254
+ registerCompletionTool();
255
+ pi.registerTool({
256
+ name: "openmerit_report_signal",
257
+ label: "Report OpenMerit Signal",
258
+ description: "Report a verified observability metric window or completed post-swap verification window to OpenMerit.",
259
+ promptSnippet: "Report verified OpenMerit observability signals",
260
+ promptGuidelines: [
261
+ "Use stable signalId values so retried deliveries remain idempotent.",
262
+ "Report only measured metrics backed by the referenced observability or evaluation artifacts.",
263
+ ],
264
+ parameters: Type.Object({
265
+ signalId: Type.String({ minLength: 1 }),
266
+ type: StringEnum(["metric_window_available", "verification_window_completed"]),
267
+ occurredAt: Type.Optional(Type.String({ minLength: 1 })),
268
+ metrics: Type.Array(MetricRecordSchema),
269
+ }, { additionalProperties: false }),
270
+ constrainedSampling: { type: "json_schema", strict: "prefer" },
271
+ async execute(_toolCallId, input, _signal, _onUpdate, ctx) {
272
+ store ??= new ProjectStore(ctx.cwd);
273
+ await reportAutomationSignal(pi, ctx, state, store, {
274
+ protocolVersion: PROTOCOL_VERSION,
275
+ id: input.signalId,
276
+ type: input.type,
277
+ occurredAt: input.occurredAt ?? new Date().toISOString(),
278
+ metrics: input.metrics,
279
+ }, registerCompletionTool);
280
+ return {
281
+ content: [{ type: "text", text: `Recorded OpenMerit automation signal ${input.signalId}.` }],
282
+ details: { signalId: input.signalId, type: input.type },
283
+ };
284
+ },
285
+ });
286
+ pi.on("session_start", async (_event, ctx) => {
287
+ store = new ProjectStore(ctx.cwd);
288
+ let projectState = await store.readState();
289
+ if (projectState) {
290
+ state.lifecycleStage = projectState.cycle.stage;
291
+ state.activeIntent = projectState.activeIntent?.id ?? null;
292
+ if (projectState.activeIntent)
293
+ registerCompletionTool(projectState.activeIntent.kind);
294
+ state.taskProfile = projectState.cycle.taskProfileId ? "configured" : "not_configured";
295
+ state.metricReadiness = projectState.observability?.status === "ready" ? "assessed" : "pending";
296
+ state.evaluationReadiness = projectState.cycle.taskProfileId ? "assessed" : "not_assessed";
297
+ const automation = await new OpenMeritCoordinator(store, piHarnessAdapter(pi)).reconcileAutomation();
298
+ state.nextAssessment = automation.plan.nextWakeupAt ??
299
+ (automation.plan.scheduling === "external_scheduler_required" ? "external scheduler required" : "signal driven");
300
+ if (automation.plan.gaps.length) {
301
+ ctx.ui.notify(`openmerit automation: ${automation.plan.gaps.join(" ")}`, "warning");
302
+ }
303
+ const fingerprint = modelCatalogFingerprint(ctx);
304
+ await reportAutomationSignal(pi, ctx, state, store, {
305
+ protocolVersion: PROTOCOL_VERSION,
306
+ id: `signal-${crypto.randomUUID()}`,
307
+ type: "model_catalog_changed",
308
+ occurredAt: new Date().toISOString(),
309
+ fingerprint,
310
+ }, registerCompletionTool);
311
+ if (!state.activeIntent) {
312
+ await reportAutomationSignal(pi, ctx, state, store, {
313
+ protocolVersion: PROTOCOL_VERSION,
314
+ id: `signal-${crypto.randomUUID()}`,
315
+ type: "scheduled_tick",
316
+ occurredAt: new Date().toISOString(),
317
+ }, registerCompletionTool);
318
+ }
319
+ }
320
+ else if (ctx.hasUI && !state.proactiveNudgesPaused) {
321
+ ctx.ui.notify("openmerit: no frontier profile found; asking Pi to establish evals and observability", "info");
322
+ await dispatchIntent(pi, ctx, state, store, "establish_evals", registerCompletionTool);
323
+ }
324
+ restoreState(ctx, state);
325
+ });
326
+ pi.on("agent_settled", async (_event, ctx) => {
327
+ if (suppressNextSettledObservation) {
328
+ suppressNextSettledObservation = false;
329
+ return;
330
+ }
331
+ store ??= new ProjectStore(ctx.cwd);
332
+ const current = await store.readState();
333
+ if (!current || current.activeIntent)
334
+ return;
335
+ await reportAutomationSignal(pi, ctx, state, store, {
336
+ protocolVersion: PROTOCOL_VERSION,
337
+ id: `signal-${crypto.randomUUID()}`,
338
+ type: "product_task_completed",
339
+ occurredAt: new Date().toISOString(),
340
+ modelId: ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : "unknown",
341
+ contextTokens: ctx.getContextUsage()?.tokens ?? null,
342
+ }, registerCompletionTool);
343
+ });
344
+ pi.registerCommand("openmerit", {
345
+ description: "OpenMerit: status, setup, assess, frontier, approve, pause/resume, logs, doctor",
346
+ handler: async (args, ctx) => {
347
+ const subcommand = args.trim().split(/\s+/)[0]?.toLowerCase() || "status";
348
+ switch (subcommand) {
349
+ case "status":
350
+ ctx.ui.notify(renderStatus(state), "info");
351
+ return;
352
+ case "setup":
353
+ store ??= new ProjectStore(ctx.cwd);
354
+ await dispatchIntent(pi, ctx, state, store, "establish_evals", registerCompletionTool);
355
+ return;
356
+ case "assess":
357
+ store ??= new ProjectStore(ctx.cwd);
358
+ await dispatchIntent(pi, ctx, state, store, "run_assessment", registerCompletionTool);
359
+ return;
360
+ case "frontier":
361
+ store ??= new ProjectStore(ctx.cwd);
362
+ await dispatchIntent(pi, ctx, state, store, "calculate_frontier", registerCompletionTool);
363
+ return;
364
+ case "approve": {
365
+ store ??= new ProjectStore(ctx.cwd);
366
+ const projectState = await store.readState();
367
+ if (!projectState?.cycle.proposedCandidateId || projectState.cycle.stage !== "frontier_ready") {
368
+ ctx.ui.notify("openmerit: there is no verified model proposal awaiting approval", "warning");
369
+ return;
370
+ }
371
+ const now = new Date().toISOString();
372
+ await store.writeState({
373
+ ...projectState,
374
+ cycle: { ...projectState.cycle, stage: "swap_approved" },
375
+ updatedAt: now,
376
+ });
377
+ await store.appendEvent({
378
+ schemaVersion: 1,
379
+ id: `event-${crypto.randomUUID()}`,
380
+ type: "cycle_advanced",
381
+ occurredAt: now,
382
+ cycleId: projectState.cycle.id,
383
+ data: { stage: "swap_approved", candidateId: projectState.cycle.proposedCandidateId },
384
+ });
385
+ state.lifecycleStage = "swap_approved";
386
+ await dispatchIntent(pi, ctx, state, store, "apply_model_swap", registerCompletionTool);
387
+ return;
388
+ }
389
+ case "pause":
390
+ state.proactiveNudgesPaused = true;
391
+ ctx.ui.notify("openmerit: proactive nudges paused", "info");
392
+ return;
393
+ case "resume":
394
+ state.proactiveNudgesPaused = false;
395
+ ctx.ui.notify("openmerit: proactive nudges resumed", "info");
396
+ return;
397
+ case "doctor":
398
+ ctx.ui.notify([
399
+ "OpenMerit doctor",
400
+ "Pi adapter : loaded",
401
+ "intent executor : connected",
402
+ `audit log : ${store?.eventsPath ?? `${ctx.cwd}/.openmerit/events.jsonl`}`,
403
+ "model mutation : disabled",
404
+ "persistent schedule: unavailable (active-session ticks only)",
405
+ `interactive UI : ${ctx.hasUI ? "available" : "unavailable"}`,
406
+ ].join("\n"), "info");
407
+ return;
408
+ case "logs":
409
+ store ??= new ProjectStore(ctx.cwd);
410
+ ctx.ui.notify([
411
+ "OpenMerit observability files",
412
+ `audit events : ${store.eventsPath}`,
413
+ `export errors: ${store.exportErrorsPath}`,
414
+ `evidence : ${store.evidenceDirectory}`,
415
+ `state : ${store.statePath}`,
416
+ ].join("\n"), "info");
417
+ return;
418
+ default:
419
+ ctx.ui.notify("openmerit: usage: /openmerit [status|setup|assess|frontier|approve|pause|resume|logs|doctor]", "warning");
420
+ }
421
+ },
422
+ });
423
+ }