@rekshaw/promptmanager 0.1.0

package/README.md

@@ -0,0 +1,117 @@
+# PromptManager
+
+PromptManager is a Node/TypeScript CLI + SDK for regression-safe prompt development with real tool-calling.
+
+## What it does
+
+- Version prompts in Git (`prompts/<promptId>/v<semver>.md`)
+- Run eval suites from JSONL fixtures
+- Execute real tool handlers in subprocess isolation
+- Validate outputs with JSON Schema + field-level assertions
+- Diff candidate runs against baseline reports and fail CI on regressions
+- Generate non-blocking prompt improvement suggestions
+
+## Install
+
+```bash
+npm install @rekshaw/promptmanager
+```
+
+## Initialize
+
+```bash
+npx promptmgr init
+```
+
+This creates:
+
+- `promptmanager.config.ts`
+- `prompts/customer-email-parser/*`
+- `evals/customer-email/*`
+- `tools/customer-email-tools.mjs`
+- `.github/workflows/promptmanager.yml`
+
+A reusable workflow template is also included at `templates/promptmanager.workflow.yml`.
+
+## CLI
+
+```bash
+promptmgr run --suite customer-email-parser --provider openai
+promptmgr diff --baseline ./baseline.json --candidate ./candidate.json
+promptmgr ci --suite customer-email-parser --provider openai --baseline ./baseline/run-report.json --fail-on-regression
+promptmgr suggest --run ./candidate.json --with-ai
+```
+
+## Config contract
+
+`promptmanager.config.ts` (or `.json`) must include:
+
+- `providers`
+- `suites`
+- `toolRunner`
+- `privacy`
+- `reporting`
+
+## Tool module contract
+
+```ts
+import type { ToolModuleShape } from "@rekshaw/promptmanager";
+
+export const tools: ToolModuleShape["tools"] = [
+  {
+    name: "my_tool",
+    description: "Tool description",
+    strict: true,
+    inputSchema: {
+      type: "object",
+      properties: { foo: { type: "string" } },
+      required: ["foo"],
+      additionalProperties: false
+    }
+  }
+];
+
+export const handlers: ToolModuleShape["handlers"] = {
+  async my_tool(args, context) {
+    return { ok: true };
+  }
+};
+```
+
+## OpenAI function-calling workflow support
+
+PromptManager's OpenAI adapter follows the same loop as the official function-calling guide:
+
+1. Send `input + tools` to `responses.create`
+2. Read `response.output` items
+3. Execute each `function_call` in your app code
+4. Append `function_call_output` items to the running `input`
+5. Call `responses.create` again until no more function calls remain
+
+For reasoning models, this preserves the full output items across turns (including reasoning/tool call items), matching the documented requirement.
+
+Customer-email parsing with fare normalization is scaffolded by default. To use your existing fare mapper, replace the local function in `tools/customer-email-tools.mjs` with an import from your codebase.
+
+## Report artifacts
+
+Each run emits a JSON artifact with:
+
+- summary counts
+- per-case status (`pass|fail|error`)
+- schema/assertion failures
+- tool-call traces
+- hashed case IDs
+
+## SDK usage
+
+```ts
+import { runSuite, diffRuns, generateSuggestions } from "@rekshaw/promptmanager";
+```
+
+## Development
+
+```bash
+npm ci
+npm run build
+npm test
+```

package/dist/assertions.d.ts

@@ -0,0 +1,3 @@
+import { AssertionResult, AssertionSpec, JsonValue } from "./types.js";
+export declare function loadAssertionSpec(assertionsPath: string): Promise<AssertionSpec>;
+export declare function evaluateAssertions(output: JsonValue, expectedOutput: JsonValue, spec: AssertionSpec): AssertionResult;

package/dist/assertions.js

@@ -0,0 +1,125 @@
+import fs from "node:fs/promises";
+import { asObject, getByPath } from "./utils.js";
+export async function loadAssertionSpec(assertionsPath) {
+    const raw = await fs.readFile(assertionsPath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed.requiredKeys)) {
+        throw new Error("Assertion spec must contain an array for 'requiredKeys'.");
+    }
+    return {
+        requiredKeys: parsed.requiredKeys,
+        allowAdditionalKeys: parsed.allowAdditionalKeys ?? false,
+        variableFields: parsed.variableFields ?? [],
+        fieldMatchers: parsed.fieldMatchers ?? {},
+    };
+}
+function resolveExpectedValue(matcher, fieldPath, expectedOutput) {
+    if (matcher.value !== undefined) {
+        return matcher.value;
+    }
+    if (matcher.expectedPath) {
+        const pathValue = matcher.expectedPath.replace(/^\$expected\./, "");
+        return getByPath(expectedOutput, pathValue);
+    }
+    return getByPath(expectedOutput, fieldPath);
+}
+function runCheck(op, actual, expected) {
+    switch (op) {
+        case "equals": {
+            const passed = JSON.stringify(actual) === JSON.stringify(expected);
+            return {
+                passed,
+                message: passed ? "matches expected value" : "value does not match expected",
+            };
+        }
+        case "oneOf": {
+            const pool = Array.isArray(expected) ? expected : [];
+            const passed = pool.some((candidate) => JSON.stringify(candidate) === JSON.stringify(actual));
+            return {
+                passed,
+                message: passed ? "value matches allowed option" : "value not in allowed set",
+            };
+        }
+        case "contains": {
+            if (typeof actual === "string" && typeof expected === "string") {
+                const passed = actual.includes(expected);
+                return { passed, message: passed ? "substring found" : "substring missing" };
+            }
+            if (Array.isArray(actual)) {
+                const passed = actual.some((item) => JSON.stringify(item) === JSON.stringify(expected));
+                return { passed, message: passed ? "array contains value" : "array does not contain value" };
+            }
+            return { passed: false, message: "contains expects string or array output" };
+        }
+        case "regex": {
+            const pattern = typeof expected === "string" ? expected : "";
+            const regex = new RegExp(pattern);
+            const passed = regex.test(String(actual ?? ""));
+            return { passed, message: passed ? "regex matched" : `regex '${pattern}' did not match` };
+        }
+        case "numericRange": {
+            const range = (expected ?? {});
+            const value = typeof actual === "number" ? actual : Number.NaN;
+            const minOk = range.min === undefined || value >= range.min;
+            const maxOk = range.max === undefined || value <= range.max;
+            const passed = Number.isFinite(value) && minOk && maxOk;
+            return {
+                passed,
+                message: passed
+                    ? "value in numeric range"
+                    : `value outside range [${range.min ?? "-inf"}, ${range.max ?? "+inf"}]`,
+            };
+        }
+        case "exists": {
+            const passed = actual !== undefined && actual !== null;
+            return { passed, message: passed ? "value exists" : "value missing" };
+        }
+        case "absent": {
+            const passed = actual === undefined || actual === null;
+            return { passed, message: passed ? "value absent as expected" : "value should be absent" };
+        }
+        default:
+            return { passed: false, message: "unsupported assertion operator" };
+    }
+}
+export function evaluateAssertions(output, expectedOutput, spec) {
+    const outputObj = asObject(output);
+    const checks = [];
+    const missingKeys = spec.requiredKeys.filter((key) => !(key in outputObj));
+    const allowedKeys = new Set([
+        ...spec.requiredKeys,
+        ...(spec.variableFields ?? []),
+        ...Object.keys(spec.fieldMatchers ?? {}),
+    ]);
+    const unexpectedKeys = (spec.allowAdditionalKeys ?? false)
+        ? []
+        : Object.keys(outputObj).filter((key) => !allowedKeys.has(key));
+    for (const [field, matchers] of Object.entries(spec.fieldMatchers ?? {})) {
+        const actualValue = getByPath(output, field);
+        for (const matcher of matchers) {
+            const expectedValue = resolveExpectedValue(matcher, field, expectedOutput);
+            const result = runCheck(matcher.op, actualValue, expectedValue);
+            const check = {
+                field,
+                op: matcher.op,
+                passed: result.passed,
+                actual: actualValue,
+                message: result.message,
+            };
+            if (expectedValue !== undefined) {
+                check.expected = expectedValue;
+            }
+            checks.push(check);
+        }
+    }
+    const passed = missingKeys.length === 0 &&
+        unexpectedKeys.length === 0 &&
+        checks.every((check) => check.passed);
+    return {
+        passed,
+        checks,
+        missingKeys,
+        unexpectedKeys,
+    };
+}
+//# sourceMappingURL=assertions.js.map

package/dist/assertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.js","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAUlC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,cAAsB;IAC5D,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAkB,CAAC;IAEhD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO;QACL,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,IAAI,KAAK;QACxD,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,EAAE;QAC3C,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAAqB,EACrB,SAAiB,EACjB,cAAyB;IAEzB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,OAAO,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;QACpE,OAAO,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,QAAQ,CACf,EAAqB,EACrB,MAA6B,EAC7B,QAAqD;IAErD,QAAQ,EAAE,EAAE,CAAC;QACX,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YACnE,OAAO;gBACL,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,+BAA+B;aAC7E,CAAC;QACJ,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YAC9F,OAAO;gBACL,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,0BAA0B;aAC9E,CAAC;QACJ,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACzC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;YAC/E,CAAC;YACD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACxF,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,8BAA8B,EAAE,CAAC;YAC/F,CAAC;YACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,yCAAyC,EAAE,CAAC;QAC/E,CAAC;QACD,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,OAAO,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YAChD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,UAAU,OAAO,iBAAiB,EAAE,CAAC;QAC5F,CAAC;QACD,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,MAAM,KAAK,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAwB,CAAC;YACtD,MAAM,KAAK,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;YAC/D,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC;YAC5D,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,KAAK,CAAC,GAAG,CAAC;YAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,KAAK,CAAC;YACxD,OAAO;gBACL,MAAM;gBACN,OAAO,EAAE,MAAM;oBACb,CAAC,CAAC,wBAAwB;oBAC1B,CAAC,CAAC,wBAAwB,KAAK,CAAC,GAAG,IAAI,MAAM,KAAK,KAAK,CAAC,GAAG,IAAI,MAAM,GAAG;aAC3E,CAAC;QACJ,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,CAAC;YACvD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC;QACxE,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,CAAC;YACvD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC,wBAAwB,EAAE,CAAC;QAC7F,CAAC;QACD;YACE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;IACxE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,MAAiB,EACjB,cAAyB,EACzB,IAAmB;IAEnB,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC;IAE3E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAS;QAClC,GAAG,IAAI,CAAC,YAAY;QACpB,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC;QAC9B,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC;KACzC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,mBAAmB,IAAI,KAAK,CAAC;QACxD,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAElE,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,EAAE,CAAC;QACzE,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,aAAa,GAAG,oBAAoB,CAAC,OAAO,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC;YAC3E,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;YAChE,MAAM,KAAK,GAAyB;gBAClC,KAAK;gBACL,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;YACF,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;gBAChC,KAAK,CAAC,QAAQ,GAAG,aAA0B,CAAC;YAC9C,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GACV,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,cAAc,CAAC,MAAM,KAAK,CAAC;QAC3B,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAExC,OAAO;QACL,MAAM;QACN,MAAM;QACN,WAAW;QACX,cAAc;KACf,CAAC;AACJ,CAAC"}

package/dist/cli/init.d.ts

@@ -0,0 +1,5 @@
+interface InitOptions {
+    force?: boolean;
+}
+export declare function runInit(cwd: string, options: InitOptions): Promise<void>;
+export {};

package/dist/cli/init.js

@@ -0,0 +1,244 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+async function writeFileSafe(file, force) {
+    await fs.mkdir(path.dirname(file.filePath), { recursive: true });
+    if (!force) {
+        try {
+            await fs.access(file.filePath);
+            return "skipped";
+        }
+        catch {
+            // file missing
+        }
+    }
+    await fs.writeFile(file.filePath, file.content, "utf8");
+    return "created";
+}
+function filesForCwd(cwd) {
+    return [
+        {
+            filePath: path.resolve(cwd, "promptmanager.config.ts"),
+            content: `import type { PromptManagerConfig } from "@rekshaw/promptmanager";
+
+const config: PromptManagerConfig = {
+  providers: {
+    openai: {
+      apiKeyEnv: "OPENAI_API_KEY",
+      parallelToolCalls: false,
+      toolChoice: "auto"
+    },
+    anthropic: { apiKeyEnv: "ANTHROPIC_API_KEY" },
+    google: { apiKeyEnv: "GEMINI_API_KEY" }
+  },
+  suites: [
+    {
+      id: "customer-email-parser",
+      promptId: "customer-email-parser",
+      datasetPath: "evals/customer-email/dataset.jsonl",
+      schemaPath: "evals/customer-email/schema.json",
+      assertionsPath: "evals/customer-email/assertions.json",
+      toolsModule: "tools/customer-email-tools.mjs",
+      modelByProvider: {
+        openai: "gpt-5-mini",
+        anthropic: "claude-3-5-sonnet-latest",
+        google: "gemini-2.0-flash"
+      }
+    }
+  ],
+  toolRunner: {
+    mode: "subprocess",
+    command: "node",
+    envAllowlist: ["TZ"],
+    timeoutMs: 5000,
+    maxToolCallsPerCase: 8
+  },
+  privacy: {
+    allowRawProductionFixtures: true,
+    redactInReports: true,
+    encryptionAtRest: false
+  },
+  reporting: {
+    includeToolTrace: true,
+    outDir: "promptmanager-reports"
+  }
+};
+
+export default config;
+`,
+        },
+        {
+            filePath: path.resolve(cwd, "prompts/customer-email-parser/meta.json"),
+            content: `${JSON.stringify({ currentVersion: "1.0.0", versions: ["1.0.0"] }, null, 2)}\n`,
+        },
+        {
+            filePath: path.resolve(cwd, "prompts/customer-email-parser/v1.0.0.md"),
+            content: `You extract structured booking data from customer confirmation emails.
+
+Rules:
+1. Return only valid JSON. No markdown. No prose.
+2. If the email contains a fare label, call the normalize_fare_type function.
+3. Set fare_type_normalized from the function output.
+4. If a field is missing, return null for that field.
+
+Required output fields:
+- customer_email
+- reservation_code
+- departure_date
+- fare_type_raw
+- fare_type_normalized
+`,
+        },
+        {
+            filePath: path.resolve(cwd, "evals/customer-email/dataset.jsonl"),
+            content: [
+                JSON.stringify({
+                    caseId: "customer-email-001",
+                    input: {
+                        subject: "Your flight is confirmed",
+                        body: "Hello Andrea, thanks for booking with us. Reservation ZX81Y is confirmed. Passenger fare type: Economy Flex. Departure date: 2026-04-18. Contact: andrea@example.com"
+                    },
+                    expected: {
+                        customer_email: "andrea@example.com",
+                        reservation_code: "ZX81Y",
+                        departure_date: "2026-04-18",
+                        fare_type_raw: "Economy Flex",
+                        fare_type_normalized: "ECONOMY_FLEX"
+                    },
+                    tags: ["happy-path", "fare-normalization"]
+                }),
+            ].join("\n") + "\n",
+        },
+        {
+            filePath: path.resolve(cwd, "evals/customer-email/schema.json"),
+            content: `${JSON.stringify({
+                type: "object",
+                properties: {
+                    customer_email: { type: ["string", "null"] },
+                    reservation_code: { type: ["string", "null"] },
+                    departure_date: { type: ["string", "null"] },
+                    fare_type_raw: { type: ["string", "null"] },
+                    fare_type_normalized: { type: ["string", "null"] }
+                },
+                required: [
+                    "customer_email",
+                    "reservation_code",
+                    "departure_date",
+                    "fare_type_raw",
+                    "fare_type_normalized"
+                ],
+                additionalProperties: false
+            }, null, 2)}\n`,
+        },
+        {
+            filePath: path.resolve(cwd, "evals/customer-email/assertions.json"),
+            content: `${JSON.stringify({
+                requiredKeys: [
+                    "customer_email",
+                    "reservation_code",
+                    "departure_date",
+                    "fare_type_raw",
+                    "fare_type_normalized"
+                ],
+                allowAdditionalKeys: false,
+                variableFields: [],
+                fieldMatchers: {
+                    customer_email: [{ op: "regex", value: "^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$" }],
+                    reservation_code: [{ op: "regex", value: "^[A-Z0-9-]{4,12}$" }],
+                    departure_date: [{ op: "regex", value: "^\\d{4}-\\d{2}-\\d{2}$" }],
+                    fare_type_normalized: [{ op: "oneOf", value: ["ECONOMY_BASIC", "ECONOMY_FLEX", "PREMIUM_ECONOMY", "BUSINESS", "FIRST", "UNKNOWN"] }]
+                }
+            }, null, 2)}\n`,
+        },
+        {
+            filePath: path.resolve(cwd, "tools/customer-email-tools.mjs"),
+            content: `// Replace this local mapper with an import from your codebase if you already have one.
+// Example: import { mapFareType } from "../src/domain/fares/mapFareType.js";
+
+const FARE_TYPE_MAP = {
+  "economy basic": "ECONOMY_BASIC",
+  "economy flex": "ECONOMY_FLEX",
+  "premium economy": "PREMIUM_ECONOMY",
+  "business": "BUSINESS",
+  "first": "FIRST"
+};
+
+function mapFareType(rawFareType) {
+  const normalized = String(rawFareType || "").trim().toLowerCase();
+  return FARE_TYPE_MAP[normalized] ?? "UNKNOWN";
+}
+
+export const tools = [
+  {
+    type: "function",
+    name: "normalize_fare_type",
+    description: "Map a raw fare label from an email into a normalized internal fare type enum",
+    strict: true,
+    inputSchema: {
+      type: "object",
+      properties: {
+        raw_fare_type: { type: "string", description: "Raw fare label, e.g. Economy Flex" }
+      },
+      required: ["raw_fare_type"],
+      additionalProperties: false
+    }
+  }
+];
+
+export const handlers = {
+  async normalize_fare_type(args) {
+    const raw = typeof args === "object" && args && "raw_fare_type" in args
+      ? String(args.raw_fare_type)
+      : "";
+
+    return {
+      raw_fare_type: raw,
+      fare_type_normalized: mapFareType(raw)
+    };
+  }
+};
+`,
+        },
+        {
+            filePath: path.resolve(cwd, ".github/workflows/promptmanager.yml"),
+            content: `name: PromptManager CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  prompt-evals:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - run: npm ci
+      - run: npm run build
+      - run: npx promptmgr ci --suite customer-email-parser --provider openai --baseline ./baseline/run-report.json --fail-on-regression
+        env:
+          OPENAI_API_KEY: \${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: \${{ secrets.ANTHROPIC_API_KEY }}
+          GEMINI_API_KEY: \${{ secrets.GEMINI_API_KEY }}
+`,
+        },
+    ];
+}
+export async function runInit(cwd, options) {
+    const files = filesForCwd(cwd);
+    let created = 0;
+    let skipped = 0;
+    for (const file of files) {
+        const outcome = await writeFileSafe(file, options.force ?? false);
+        if (outcome === "created") {
+            created += 1;
+        }
+        else {
+            skipped += 1;
+        }
+    }
+    process.stdout.write(`Initialized PromptManager scaffold. created=${created} skipped=${skipped}\n`);
+}
+//# sourceMappingURL=init.js.map

package/dist/cli/init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAW7B,KAAK,UAAU,aAAa,CAAC,IAAkB,EAAE,KAAc;IAC7D,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACjE,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/B,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;IACH,CAAC;IACD,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACxD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO;QACL;YACE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,yBAAyB,CAAC;YACtD,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Cd;SACI;QACD;YACE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,yCAAyC,CAAC;YACtE,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI;SAC1F;QACD;YACE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,yCAAyC,CAAC;YACtE,OAAO,EAAE;;;;;;;;;;;;;;CAcd;SACI;QACD;YACE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,oCAAoC,CAAC;YACjE,OAAO,EAAE;gBACP,IAAI,CAAC,SAAS,CAAC;oBACb,MAAM,EAAE,oBAAoB;oBAC5B,KAAK,EAAE;wBACL,OAAO,EAAE,0BAA0B;wBACnC,IAAI,EAAE,sKAAsK;qBAC7K;oBACD,QAAQ,EAAE;wBACR,cAAc,EAAE,oBAAoB;wBACpC,gBAAgB,EAAE,OAAO;wBACzB,cAAc,EAAE,YAAY;wBAC5B,aAAa,EAAE,cAAc;wBAC7B,oBAAoB,EAAE,cAAc;qBACrC;oBACD,IAAI,EAAE,CAAC,YAAY,EAAE,oBAAoB,CAAC;iBAC3C,CAAC;aACH,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI;SACpB;QACD;YACE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,kCAAkC,CAAC;YAC/D,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CACxB;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,cAAc,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE;oBAC5C,gBAAgB,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE;oBAC9C,cAAc,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE;oBAC5C,aAAa,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE;oBAC3C,oBAAoB,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE;iBACnD;gBACD,QAAQ,EAAE;oBACR,gBAAgB;oBAChB,kBAAkB;oBAClB,gBAAgB;oBAChB,eAAe;oBACf,sBAAsB;iBACvB;gBACD,oBAAoB,EAAE,KAAK;aAC5B,EACD,IAAI,EACJ,CAAC,CACF,IAAI;SACN;QACD;YACE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,sCAAsC,CAAC;YACnE,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CACxB;gBACE,YAAY,EAAE;oBACZ,gBAAgB;oBAChB,kBAAkB;oBAClB,gBAAgB;oBAChB,eAAe;oBACf,sBAAsB;iBACvB;gBACD,mBAAmB,EAAE,KAAK;gBAC1B,cAAc,EAAE,EAAE;gBAClB,aAAa,EAAE;oBACb,cAAc,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;oBAC1E,gBAAgB,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,mBAAmB,EAAE,CAAC;oBAC/D,cAAc,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;oBAClE,oBAAoB,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,eAAe,EAAE,cAAc,EAAE,iBAAiB,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;iBACrI;aACF,EACD,IAAI,EACJ,CAAC,CACF,IAAI;SACN;QACD;YACE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,gCAAgC,CAAC;YAC7D,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Cd;SACI;QACD;YACE,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,qCAAqC,CAAC;YAClE,OAAO,EAAE;;;;;;;;;;;;;;;;;;;;;;CAsBd;SACI;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,GAAW,EAAE,OAAoB;IAC7D,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC;QAClE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+CAA+C,OAAO,YAAY,OAAO,IAAI,CAAC,CAAC;AACtG,CAAC"}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+import path from "node:path";
+import { Command } from "commander";
+import { runSuite } from "./runSuite.js";
+import { diffRuns } from "./diffRuns.js";
+import { generateSuggestions } from "./suggestions.js";
+import { runInit } from "./cli/init.js";
+import { loadConfig } from "./config.js";
+import { defaultRunReportPath, defaultSuggestionPath, printDiffSummary, printRunSummary, printSuggestionSummary, writeRunReport, writeSuggestionReport, } from "./reporting.js";
+import { readJsonFile } from "./utils.js";
+function parseProvider(value) {
+    if (value === "openai" || value === "anthropic" || value === "google") {
+        return value;
+    }
+    throw new Error(`Invalid provider '${value}'. Expected openai|anthropic|google.`);
+}
+function toInt(value, fallback) {
+    const parsed = Number.parseInt(value, 10);
+    return Number.isFinite(parsed) ? parsed : fallback;
+}
+async function resolveConfiguredOutDir(cwd, configPath) {
+    try {
+        const { path: loadedPath, config } = await loadConfig(cwd, configPath);
+        const baseDir = path.dirname(loadedPath);
+        return path.resolve(baseDir, config.reporting.outDir ?? "promptmanager-reports");
+    }
+    catch {
+        return path.resolve(cwd, "promptmanager-reports");
+    }
+}
+async function main() {
+    const program = new Command();
+    program
+        .name("promptmgr")
+        .description("PromptManager: regression-safe prompt + tool-calling evaluation")
+        .version("0.1.0");
+    program
+        .command("init")
+        .description("Initialize PromptManager scaffold in current repo")
+        .option("--force", "Overwrite existing scaffold files")
+        .action(async (options) => {
+        await runInit(process.cwd(), options);
+    });
+    program
+        .command("run")
+        .description("Run one suite against a provider/model and emit run report")
+        .requiredOption("--suite <suite>", "Suite ID")
+        .requiredOption("--provider <provider>", "Provider: openai|anthropic|google")
+        .option("--model <model>", "Model override")
+        .option("--out <path>", "Output report JSON path")
+        .option("--config <path>", "Config path (default: promptmanager.config.ts/json)")
+        .option("--concurrency <n>", "Parallel case workers (default: 4)", "4")
+        .action(async (options) => {
+        const provider = parseProvider(options.provider);
+        const report = await runSuite({
+            suiteId: options.suite,
+            provider,
+            model: options.model,
+            outPath: options.out,
+            configPath: options.config,
+            cwd: process.cwd(),
+            concurrency: toInt(options.concurrency, 4),
+        });
+        printRunSummary(report);
+        const outDir = await resolveConfiguredOutDir(process.cwd(), options.config);
+        const outPath = options.out
+            ? path.resolve(process.cwd(), options.out)
+            : defaultRunReportPath(process.cwd(), options.suite, provider, outDir);
+        await writeRunReport(outPath, report);
+        process.stdout.write(`Run report written: ${outPath}\n`);
+    });
+    program
+        .command("diff")
+        .description("Diff baseline vs candidate run reports")
+        .requiredOption("--baseline <path>", "Baseline report path")
+        .requiredOption("--candidate <path>", "Candidate report path")
+        .action(async (options) => {
+        const baseline = await readJsonFile(path.resolve(process.cwd(), options.baseline));
+        const candidate = await readJsonFile(path.resolve(process.cwd(), options.candidate));
+        const diff = diffRuns(baseline, candidate);
+        printDiffSummary(diff);
+        process.stdout.write(`${JSON.stringify(diff, null, 2)}\n`);
+    });
+    program
+        .command("ci")
+        .description("Run suite and fail on regression against a baseline")
+        .requiredOption("--suite <suite>", "Suite ID")
+        .requiredOption("--provider <provider>", "Provider: openai|anthropic|google")
+        .requiredOption("--baseline <path>", "Baseline report path")
+        .option("--model <model>", "Model override")
+        .option("--config <path>", "Config path")
+        .option("--out <path>", "Candidate report output path")
+        .option("--fail-on-regression", "Fail CI when regressions exist", true)
+        .option("--concurrency <n>", "Parallel case workers (default: 4)", "4")
+        .action(async (options) => {
+        const provider = parseProvider(options.provider);
+        const candidate = await runSuite({
+            suiteId: options.suite,
+            provider,
+            model: options.model,
+            configPath: options.config,
+            cwd: process.cwd(),
+            concurrency: toInt(options.concurrency, 4),
+        });
+        printRunSummary(candidate);
+        const outDir = await resolveConfiguredOutDir(process.cwd(), options.config);
+        const candidatePath = options.out
+            ? path.resolve(process.cwd(), options.out)
+            : defaultRunReportPath(process.cwd(), options.suite, provider, outDir);
+        await writeRunReport(candidatePath, candidate);
+        process.stdout.write(`Candidate run report written: ${candidatePath}\n`);
+        const baseline = await readJsonFile(path.resolve(process.cwd(), options.baseline));
+        const diff = diffRuns(baseline, candidate);
+        printDiffSummary(diff);
+        if ((options.failOnRegression ?? true) && diff.regressions.length > 0) {
+            process.stderr.write(`CI failed: ${diff.regressions.length} regressions detected.\n`);
+            process.exitCode = 1;
+            return;
+        }
+        process.stdout.write("CI check passed.\n");
+    });
+    program
+        .command("suggest")
+        .description("Generate prompt improvement suggestions from a run report")
+        .requiredOption("--run <path>", "Run report path")
+        .option("--out <path>", "Output suggestion report path")
+        .option("--with-ai", "Use AI suggestion generation when OPENAI_API_KEY is available", false)
+        .option("--model <model>", "Suggestion model override")
+        .option("--max <n>", "Max number of suggestions", "5")
+        .action(async (options) => {
+        const report = await readJsonFile(path.resolve(process.cwd(), options.run));
+        const suggestions = await generateSuggestions({
+            report,
+            maxSuggestions: toInt(options.max, 5),
+            withAi: options.withAi ?? false,
+            aiModel: options.model,
+        });
+        printSuggestionSummary(suggestions);
+        const outPath = options.out
+            ? path.resolve(process.cwd(), options.out)
+            : defaultSuggestionPath(process.cwd(), report.suiteId);
+        await writeSuggestionReport(outPath, suggestions);
+        process.stdout.write(`Suggestion report written: ${outPath}\n`);
+    });
+    await program.parseAsync(process.argv);
+}
+main().catch((error) => {
+    process.stderr.write(`promptmgr failed: ${error instanceof Error ? error.message : String(error)}\n`);
+    process.exitCode = 1;
+});
+//# sourceMappingURL=cli.js.map