@elizaos/plugin-training 2.0.3-beta.5 → 2.0.3-beta.7

package/dist/core/privacy-filter.js

@@ -0,0 +1,324 @@
+import { createHash } from "node:crypto";
+const DEFAULT_PLATFORMS = [
+  "telegram",
+  "discord",
+  "slack",
+  "matrix",
+  "signal",
+  "whatsapp",
+  "twitter",
+  "instagram",
+  "email"
+];
+const HANDLE_PATTERN = /(@[a-zA-Z0-9_.-]{2,})/g;
+const DEFAULT_CREDENTIAL_PATTERNS = [
+  // `sk-ant-…` must be matched before the generic `sk-…` so the more specific
+  // Anthropic label wins.
+  { label: "anthropic-key", pattern: /\bsk-ant-[A-Za-z0-9_-]{16,}\b/g },
+  { label: "openai-key", pattern: /\bsk-[A-Za-z0-9_-]{16,}\b/g },
+  {
+    label: "bearer",
+    pattern: /\bBearer\s+[A-Za-z0-9._-]{16,}\b/g
+  },
+  {
+    label: "github-token",
+    pattern: /\bghp_[A-Za-z0-9]{20,}\b/g
+  },
+  {
+    label: "aws-access-key",
+    pattern: /\bAKIA[0-9A-Z]{16}\b/g
+  }
+];
+const EMAIL_REPLACEMENT = "[REDACTED_EMAIL]";
+const PHONE_REPLACEMENT = "[REDACTED_PHONE]";
+const ADDRESS_REPLACEMENT = "[REDACTED_ADDRESS]";
+const EMAIL_PATTERN = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;
+const STREET_SUFFIXES = "St|Street|Ave|Avenue|Blvd|Boulevard|Rd|Road|Ln|Lane|Dr|Drive|Ct|Court|Pl|Place|Way|Pkwy|Parkway|Ter|Terrace|Cir|Circle|Hwy|Highway|Sq|Square|Trl|Trail|Loop";
+const UNIT_DESIGNATORS = "Apt|Apartment|Suite|Ste|Unit|Bldg|Building|Fl|Floor|Rm|Room|#";
+const US_STATES = "AL|AK|AZ|AR|CA|CO|CT|DE|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY|DC";
+const DEFAULT_ADDRESS_PATTERNS = [
+  // 1. Numbered street + suffix + optional unit, optionally followed by a
+  //    city, state, ZIP tail: `1600 Amphitheatre Parkway, Suite 200,
+  //    Mountain View, CA 94043`.
+  new RegExp(
+    String.raw`\b\d{1,6}\s+(?:[A-Za-z0-9.'-]+\s+){0,4}(?:${STREET_SUFFIXES})\b` + String.raw`(?:\s*,?\s*(?:${UNIT_DESIGNATORS})\.?\s*[A-Za-z0-9-]+)?` + String.raw`(?:\s*,\s*[A-Za-z .'-]+,?\s*(?:${US_STATES})\s+\d{5}(?:-\d{4})?)?`,
+    "gi"
+  ),
+  // 2. `PO Box 4242` / `P.O. Box 4242`.
+  /\bP\.?\s?O\.?\s?Box\s+\d{1,7}\b/gi,
+  // 3. Standalone city, state, ZIP tail: `Mountain View, CA 94043`.
+  new RegExp(
+    String.raw`\b[A-Za-z .'-]+,\s*(?:${US_STATES})\s+\d{5}(?:-\d{4})?\b`,
+    "g"
+  )
+];
+const DEFAULT_PHONE_PATTERNS = [
+  // 1. E.164 / international with leading `+`: `+44 20 7946 0958`,
+  //    `+1-415-555-0123`, `+442079460958`.
+  /\+\d{1,3}(?:[\s.-]?\d{1,4}){1,5}\b/g,
+  // 2. NANP with explicit separators (a separator is REQUIRED between groups
+  //    so bare 10-digit runs survive): `(415) 555-0123`, `415-555-0123`,
+  //    `415.555.0123`, `415 555 0123`. No leading `\b` before `(` — there is
+  //    no word boundary between a space and `(`.
+  /(?:\(\d{3}\)[\s.-]?|\b\d{3}[\s.-])\d{3}[\s.-]\d{4}\b/g
+];
+const GEO_REPLACEMENT = "[REDACTED_GEO]";
+const DEFAULT_GEO_PATTERNS = [
+  // 1. JSON `"coords":{"latitude":..,"longitude":..[,...]}` (Capacitor shape).
+  /"coords"\s*:\s*\{\s*"latitude"\s*:\s*-?\d+(?:\.\d+)?\s*,\s*"longitude"\s*:\s*-?\d+(?:\.\d+)?(?:\s*,\s*"[A-Za-z_][A-Za-z0-9_]*"\s*:\s*[^,}]+)*\s*\}/g,
+  // 2. Bare JSON pair `"latitude":..,"longitude":..`.
+  /"latitude"\s*:\s*-?\d+(?:\.\d+)?\s*,\s*"longitude"\s*:\s*-?\d+(?:\.\d+)?/g,
+  // 3. `current location: 37.7, -122.4` / `coords: ...` / `coordinates=...`.
+  /\b(?:current\s+location|location|coords|coordinates)\s*[:=]\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?/gi,
+  // 4. Labeled `lat: .., lng: ..` / `latitude=.., longitude=..`.
+  /\b(?:lat|latitude)\s*[:=]\s*-?\d+(?:\.\d+)?\s*[,;]\s*(?:lng|lon|long|longitude)\s*[:=]\s*-?\d+(?:\.\d+)?/gi,
+  // 5. Bare decimal pair `37.7749, -122.4194` (both numbers must have a
+  //    fractional component to avoid matching integer pairs).
+  /\b-?\d{1,3}\.\d{2,}\s*,\s*-?\d{1,3}\.\d{2,}\b/g
+];
+function snapshotEnvCredentials(envKeys) {
+  const interesting = /KEY|TOKEN|SECRET|PASSWORD|API|CREDENTIAL/i;
+  const out = [];
+  for (const key of envKeys) {
+    if (!interesting.test(key)) continue;
+    const value = process.env[key];
+    if (typeof value !== "string") continue;
+    if (value.length < 8) continue;
+    out.push(value);
+  }
+  return out;
+}
+function redactCredentials(value, patterns, credentialValues, state) {
+  let out = value;
+  for (const { label, pattern } of patterns) {
+    out = out.replace(pattern, () => {
+      state.redactionCount += 1;
+      return `<REDACTED:${label}>`;
+    });
+  }
+  for (const credValue of credentialValues) {
+    if (!credValue) continue;
+    const escaped = credValue.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const re = new RegExp(escaped, "g");
+    out = out.replace(re, () => {
+      state.redactionCount += 1;
+      return "<REDACTED:env-secret>";
+    });
+  }
+  return out;
+}
+function redactGeo(value, state) {
+  let out = value;
+  for (const pattern of DEFAULT_GEO_PATTERNS) {
+    out = out.replace(pattern, () => {
+      state.redactionCount += 1;
+      return GEO_REPLACEMENT;
+    });
+  }
+  return out;
+}
+function redactPii(value, state) {
+  let out = value;
+  out = out.replace(EMAIL_PATTERN, () => {
+    state.redactionCount += 1;
+    return EMAIL_REPLACEMENT;
+  });
+  for (const pattern of DEFAULT_ADDRESS_PATTERNS) {
+    out = out.replace(pattern, () => {
+      state.redactionCount += 1;
+      return ADDRESS_REPLACEMENT;
+    });
+  }
+  for (const pattern of DEFAULT_PHONE_PATTERNS) {
+    out = out.replace(pattern, () => {
+      state.redactionCount += 1;
+      return PHONE_REPLACEMENT;
+    });
+  }
+  return out;
+}
+function anonymizeHandles(value, options, state) {
+  const platforms = options.platforms ?? DEFAULT_PLATFORMS;
+  const entityHits = /* @__PURE__ */ new Set();
+  if (!options.anonymizer) {
+    return { result: value, entityHits };
+  }
+  const result = value.replace(HANDLE_PATTERN, (match, handle) => {
+    const stripped = handle.startsWith("@") ? handle.slice(1) : handle;
+    for (const platform of platforms) {
+      const entityId = options.anonymizer?.resolveEntityId(platform, stripped);
+      if (entityId) {
+        state.anonymizationCount += 1;
+        entityHits.add(entityId);
+        return `<entity:${entityId}>`;
+      }
+    }
+    return match;
+  });
+  return { result, entityHits };
+}
+function transformText(value, options, credentialValues, credentialPatterns, state, collectedEntities) {
+  const geoRedacted = redactGeo(value, state);
+  const credRedacted = redactCredentials(
+    geoRedacted,
+    credentialPatterns,
+    credentialValues,
+    state
+  );
+  const piiRedacted = redactPii(credRedacted, state);
+  const { result, entityHits } = anonymizeHandles(piiRedacted, options, state);
+  for (const entityId of entityHits) collectedEntities.add(entityId);
+  return result;
+}
+function transformDeep(value, options, credentialValues, credentialPatterns, state, collectedEntities) {
+  if (typeof value === "string") {
+    return transformText(
+      value,
+      options,
+      credentialValues,
+      credentialPatterns,
+      state,
+      collectedEntities
+    );
+  }
+  if (Array.isArray(value)) {
+    return value.map(
+      (entry) => transformDeep(
+        entry,
+        options,
+        credentialValues,
+        credentialPatterns,
+        state,
+        collectedEntities
+      )
+    );
+  }
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [key, entry] of Object.entries(
+      value
+    )) {
+      out[key] = transformDeep(
+        entry,
+        options,
+        credentialValues,
+        credentialPatterns,
+        state,
+        collectedEntities
+      );
+    }
+    return out;
+  }
+  return value;
+}
+function applyPrivacyFilter(trajectories, options = {}) {
+  const credentialPatterns = [
+    ...DEFAULT_CREDENTIAL_PATTERNS,
+    ...options.extraCredentialPatterns ?? []
+  ];
+  const envKeys = options.envKeySnapshot ?? Object.keys(process.env);
+  const credentialValues = snapshotEnvCredentials(envKeys);
+  const dropped = [];
+  const filtered = [];
+  const state = {
+    anonymizationCount: 0,
+    redactionCount: 0
+  };
+  for (const trajectory of trajectories) {
+    const trajectoryEntities = /* @__PURE__ */ new Set();
+    const cloned = JSON.parse(JSON.stringify(trajectory));
+    const steps = cloned.steps ?? [];
+    for (const step of steps) {
+      for (const call of step.llmCalls ?? []) {
+        if (typeof call.systemPrompt === "string") {
+          call.systemPrompt = transformText(
+            call.systemPrompt,
+            options,
+            credentialValues,
+            credentialPatterns,
+            state,
+            trajectoryEntities
+          );
+        }
+        if (typeof call.userPrompt === "string") {
+          call.userPrompt = transformText(
+            call.userPrompt,
+            options,
+            credentialValues,
+            credentialPatterns,
+            state,
+            trajectoryEntities
+          );
+        }
+        if (typeof call.response === "string") {
+          call.response = transformText(
+            call.response,
+            options,
+            credentialValues,
+            credentialPatterns,
+            state,
+            trajectoryEntities
+          );
+        }
+      }
+      for (const access of step.providerAccesses ?? []) {
+        if (access.data !== void 0) {
+          access.data = transformDeep(
+            access.data,
+            options,
+            credentialValues,
+            credentialPatterns,
+            state,
+            trajectoryEntities
+          );
+        }
+      }
+    }
+    if (cloned.metadata && typeof cloned.metadata === "object") {
+      cloned.metadata = transformDeep(
+        cloned.metadata,
+        options,
+        credentialValues,
+        credentialPatterns,
+        state,
+        trajectoryEntities
+      );
+    }
+    const lookup = options.anonymizer?.getPrivacyLevel;
+    if (lookup) {
+      let isPrivate = false;
+      for (const entityId of trajectoryEntities) {
+        if (lookup(entityId) === "private") {
+          isPrivate = true;
+          break;
+        }
+      }
+      if (isPrivate) {
+        dropped.push({
+          trajectoryId: trajectory.trajectoryId,
+          reason: "entity-private"
+        });
+        continue;
+      }
+    }
+    filtered.push(cloned);
+  }
+  return {
+    trajectories: filtered,
+    dropped,
+    redactionCount: state.redactionCount,
+    anonymizationCount: state.anonymizationCount
+  };
+}
+function createHashAnonymizer(salt = "") {
+  return {
+    resolveEntityId(platform, handle) {
+      return createHash("sha256").update(`${salt}:${platform.toLowerCase()}:${handle.toLowerCase()}`).digest("hex").slice(0, 16);
+    }
+  };
+}
+export {
+  applyPrivacyFilter,
+  createHashAnonymizer
+};
+//# sourceMappingURL=privacy-filter.js.map

package/dist/core/privacy-filter.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/privacy-filter.ts"],"sourcesContent":["/**\n * Privacy filter for trajectory exports.\n *\n * Jobs:\n *   1. Anonymize cross-platform handles by mapping them to opaque entity IDs\n *      (the caller supplies a lookup callback so app-training does not have\n *      to depend on the relationships service directly). `createHashAnonymizer`\n *      provides a stable, dependency-free default.\n *   2. Honor `ContactPreferences.privacyLevel` — drop entire trajectories if\n *      the participating entity is `private`.\n *   3. Strip credential references — env-var name patterns from process.env,\n *      plus the usual API key shapes (`sk-ant-…`, `sk-…`, `Bearer …`).\n *   4. Strip geo coordinates — bare decimal pairs, labeled `lat:`/`lng:`\n *      values, and JSON `\"coords\":{\"latitude\":..,\"longitude\":..}` blocks\n *      from the Location plugin — replaced with `[REDACTED_GEO]`.\n *   5. Strip PII — email addresses (`[REDACTED_EMAIL]`), phone numbers\n *      (`[REDACTED_PHONE]`), and street/PO-box/city-state-ZIP addresses\n *      (`[REDACTED_ADDRESS]`).\n *\n * Walks every string in `steps[].llmCalls[].{systemPrompt,userPrompt,response}`,\n * `steps[].providerAccesses[].data`, and the top-level `metadata` object.\n *\n * Run automatically before any export to disk; required for any cloud upload.\n */\n\nimport { createHash } from \"node:crypto\";\n\nexport type PrivacyLevel = \"public\" | \"limited\" | \"private\";\n\nexport interface AnonymizerLookup {\n  /** Look up the opaque entity ID for a (platform, handle) pair. */\n  resolveEntityId(platform: string, handle: string): string | null;\n  /** Look up the privacy level for an entity. Defaults to \"public\". */\n  getPrivacyLevel?(entityId: string): PrivacyLevel | undefined;\n}\n\nexport interface PrivacyFilterOptions {\n  /** Optional anonymizer lookup. If absent, handles pass through unchanged. */\n  anonymizer?: AnonymizerLookup;\n  /**\n   * Additional credential shapes to redact. Each entry is matched as a\n   * RegExp against any string field; matches are replaced with\n   * `<REDACTED:{label}>`.\n   */\n  extraCredentialPatterns?: Array<{ label: string; pattern: RegExp }>;\n  /**\n   * Snapshot of `process.env` keys to treat as credential names.\n   * Defaults to capturing all env names matching the standard secret regex.\n   */\n  envKeySnapshot?: string[];\n  /**\n   * Hard list of platforms the anonymizer recognizes. Used to constrain\n   * cross-platform handle detection. Defaults to common platforms.\n   */\n  platforms?: string[];\n}\n\nexport interface FilterableTrajectory {\n  trajectoryId?: string;\n  steps?: Array<{\n    llmCalls?: Array<{\n      systemPrompt?: string;\n      userPrompt?: string;\n      response?: string;\n    }>;\n    providerAccesses?: Array<{\n      data?: unknown;\n    }>;\n  }>;\n  metadata?: Record<string, unknown>;\n  [key: string]: unknown;\n}\n\nexport interface FilterResult<T> {\n  trajectories: T[];\n  dropped: Array<{ trajectoryId?: string; reason: string }>;\n  redactionCount: number;\n  anonymizationCount: number;\n}\n\nconst DEFAULT_PLATFORMS = [\n  \"telegram\",\n  \"discord\",\n  \"slack\",\n  \"matrix\",\n  \"signal\",\n  \"whatsapp\",\n  \"twitter\",\n  \"instagram\",\n  \"email\",\n];\n\nconst HANDLE_PATTERN = /(@[a-zA-Z0-9_.-]{2,})/g;\n\nconst DEFAULT_CREDENTIAL_PATTERNS: Array<{ label: string; pattern: RegExp }> = [\n  // `sk-ant-…` must be matched before the generic `sk-…` so the more specific\n  // Anthropic label wins.\n  { label: \"anthropic-key\", pattern: /\\bsk-ant-[A-Za-z0-9_-]{16,}\\b/g },\n  { label: \"openai-key\", pattern: /\\bsk-[A-Za-z0-9_-]{16,}\\b/g },\n  {\n    label: \"bearer\",\n    pattern: /\\bBearer\\s+[A-Za-z0-9._-]{16,}\\b/g,\n  },\n  {\n    label: \"github-token\",\n    pattern: /\\bghp_[A-Za-z0-9]{20,}\\b/g,\n  },\n  {\n    label: \"aws-access-key\",\n    pattern: /\\bAKIA[0-9A-Z]{16}\\b/g,\n  },\n];\n\n/**\n * PII redaction (email / phone / address). Applied in the order\n * email → address → phone so phone-like number runs inside an address tail\n * (e.g. ZIP codes) are consumed by the address pass first, and bare digit\n * runs without separators survive.\n */\nconst EMAIL_REPLACEMENT = \"[REDACTED_EMAIL]\";\nconst PHONE_REPLACEMENT = \"[REDACTED_PHONE]\";\nconst ADDRESS_REPLACEMENT = \"[REDACTED_ADDRESS]\";\n\nconst EMAIL_PATTERN = /\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}\\b/g;\n\nconst STREET_SUFFIXES =\n  \"St|Street|Ave|Avenue|Blvd|Boulevard|Rd|Road|Ln|Lane|Dr|Drive|Ct|Court|Pl|Place|Way|Pkwy|Parkway|Ter|Terrace|Cir|Circle|Hwy|Highway|Sq|Square|Trl|Trail|Loop\";\nconst UNIT_DESIGNATORS =\n  \"Apt|Apartment|Suite|Ste|Unit|Bldg|Building|Fl|Floor|Rm|Room|#\";\nconst US_STATES =\n  \"AL|AK|AZ|AR|CA|CO|CT|DE|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY|DC\";\n\nconst DEFAULT_ADDRESS_PATTERNS: RegExp[] = [\n  // 1. Numbered street + suffix + optional unit, optionally followed by a\n  //    city, state, ZIP tail: `1600 Amphitheatre Parkway, Suite 200,\n  //    Mountain View, CA 94043`.\n  new RegExp(\n    String.raw`\\b\\d{1,6}\\s+(?:[A-Za-z0-9.'-]+\\s+){0,4}(?:${STREET_SUFFIXES})\\b` +\n      String.raw`(?:\\s*,?\\s*(?:${UNIT_DESIGNATORS})\\.?\\s*[A-Za-z0-9-]+)?` +\n      String.raw`(?:\\s*,\\s*[A-Za-z .'-]+,?\\s*(?:${US_STATES})\\s+\\d{5}(?:-\\d{4})?)?`,\n    \"gi\",\n  ),\n  // 2. `PO Box 4242` / `P.O. Box 4242`.\n  /\\bP\\.?\\s?O\\.?\\s?Box\\s+\\d{1,7}\\b/gi,\n  // 3. Standalone city, state, ZIP tail: `Mountain View, CA 94043`.\n  new RegExp(\n    String.raw`\\b[A-Za-z .'-]+,\\s*(?:${US_STATES})\\s+\\d{5}(?:-\\d{4})?\\b`,\n    \"g\",\n  ),\n];\n\nconst DEFAULT_PHONE_PATTERNS: RegExp[] = [\n  // 1. E.164 / international with leading `+`: `+44 20 7946 0958`,\n  //    `+1-415-555-0123`, `+442079460958`.\n  /\\+\\d{1,3}(?:[\\s.-]?\\d{1,4}){1,5}\\b/g,\n  // 2. NANP with explicit separators (a separator is REQUIRED between groups\n  //    so bare 10-digit runs survive): `(415) 555-0123`, `415-555-0123`,\n  //    `415.555.0123`, `415 555 0123`. No leading `\\b` before `(` — there is\n  //    no word boundary between a space and `(`.\n  /(?:\\(\\d{3}\\)[\\s.-]?|\\b\\d{3}[\\s.-])\\d{3}[\\s.-]\\d{4}\\b/g,\n];\n\n/**\n * Geo coordinate redaction.\n *\n * The travel-time consumer now reads from the Location plugin\n * (`plugins/plugin-personal-assistant/src/travel-time/service.ts`), so precise lat/lon\n * values can land in trajectory text. We strip them before any export with\n * the marker `[REDACTED_GEO]`.\n *\n * Patterns are intentionally narrow — they require a lat/lng label, a JSON\n * wrapper, or at least one decimal place per number — so we do not redact\n * ordinary integer pairs (timestamps, IDs) that happen to be comma-separated.\n *\n * Order matters: the JSON `coords` block is consumed first so the inner\n * `latitude/longitude` pair does not get redacted twice.\n */\nconst GEO_REPLACEMENT = \"[REDACTED_GEO]\";\n\nconst DEFAULT_GEO_PATTERNS: RegExp[] = [\n  // 1. JSON `\"coords\":{\"latitude\":..,\"longitude\":..[,...]}` (Capacitor shape).\n  /\"coords\"\\s*:\\s*\\{\\s*\"latitude\"\\s*:\\s*-?\\d+(?:\\.\\d+)?\\s*,\\s*\"longitude\"\\s*:\\s*-?\\d+(?:\\.\\d+)?(?:\\s*,\\s*\"[A-Za-z_][A-Za-z0-9_]*\"\\s*:\\s*[^,}]+)*\\s*\\}/g,\n  // 2. Bare JSON pair `\"latitude\":..,\"longitude\":..`.\n  /\"latitude\"\\s*:\\s*-?\\d+(?:\\.\\d+)?\\s*,\\s*\"longitude\"\\s*:\\s*-?\\d+(?:\\.\\d+)?/g,\n  // 3. `current location: 37.7, -122.4` / `coords: ...` / `coordinates=...`.\n  /\\b(?:current\\s+location|location|coords|coordinates)\\s*[:=]\\s*-?\\d+(?:\\.\\d+)?\\s*,\\s*-?\\d+(?:\\.\\d+)?/gi,\n  // 4. Labeled `lat: .., lng: ..` / `latitude=.., longitude=..`.\n  /\\b(?:lat|latitude)\\s*[:=]\\s*-?\\d+(?:\\.\\d+)?\\s*[,;]\\s*(?:lng|lon|long|longitude)\\s*[:=]\\s*-?\\d+(?:\\.\\d+)?/gi,\n  // 5. Bare decimal pair `37.7749, -122.4194` (both numbers must have a\n  //    fractional component to avoid matching integer pairs).\n  /\\b-?\\d{1,3}\\.\\d{2,}\\s*,\\s*-?\\d{1,3}\\.\\d{2,}\\b/g,\n];\n\nfunction snapshotEnvCredentials(envKeys: string[]): string[] {\n  // Heuristic: a key counts as a credential if its NAME matches a common\n  // secret-marker substring AND its VALUE is non-empty and reasonably long.\n  const interesting = /KEY|TOKEN|SECRET|PASSWORD|API|CREDENTIAL/i;\n  const out: string[] = [];\n  for (const key of envKeys) {\n    if (!interesting.test(key)) continue;\n    const value = process.env[key];\n    if (typeof value !== \"string\") continue;\n    if (value.length < 8) continue;\n    out.push(value);\n  }\n  return out;\n}\n\ninterface InternalState {\n  anonymizationCount: number;\n  redactionCount: number;\n}\n\nfunction redactCredentials(\n  value: string,\n  patterns: Array<{ label: string; pattern: RegExp }>,\n  credentialValues: string[],\n  state: InternalState,\n): string {\n  let out = value;\n  for (const { label, pattern } of patterns) {\n    out = out.replace(pattern, () => {\n      state.redactionCount += 1;\n      return `<REDACTED:${label}>`;\n    });\n  }\n  for (const credValue of credentialValues) {\n    if (!credValue) continue;\n    const escaped = credValue.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n    const re = new RegExp(escaped, \"g\");\n    out = out.replace(re, () => {\n      state.redactionCount += 1;\n      return \"<REDACTED:env-secret>\";\n    });\n  }\n  return out;\n}\n\nfunction redactGeo(value: string, state: InternalState): string {\n  let out = value;\n  for (const pattern of DEFAULT_GEO_PATTERNS) {\n    out = out.replace(pattern, () => {\n      state.redactionCount += 1;\n      return GEO_REPLACEMENT;\n    });\n  }\n  return out;\n}\n\nfunction redactPii(value: string, state: InternalState): string {\n  let out = value;\n  // email → address → phone (see note on DEFAULT_PHONE_PATTERNS / addresses).\n  out = out.replace(EMAIL_PATTERN, () => {\n    state.redactionCount += 1;\n    return EMAIL_REPLACEMENT;\n  });\n  for (const pattern of DEFAULT_ADDRESS_PATTERNS) {\n    out = out.replace(pattern, () => {\n      state.redactionCount += 1;\n      return ADDRESS_REPLACEMENT;\n    });\n  }\n  for (const pattern of DEFAULT_PHONE_PATTERNS) {\n    out = out.replace(pattern, () => {\n      state.redactionCount += 1;\n      return PHONE_REPLACEMENT;\n    });\n  }\n  return out;\n}\n\nfunction anonymizeHandles(\n  value: string,\n  options: PrivacyFilterOptions,\n  state: InternalState,\n): { result: string; entityHits: Set<string> } {\n  const platforms = options.platforms ?? DEFAULT_PLATFORMS;\n  const entityHits = new Set<string>();\n  if (!options.anonymizer) {\n    return { result: value, entityHits };\n  }\n\n  const result = value.replace(HANDLE_PATTERN, (match, handle: string) => {\n    const stripped = handle.startsWith(\"@\") ? handle.slice(1) : handle;\n    for (const platform of platforms) {\n      const entityId = options.anonymizer?.resolveEntityId(platform, stripped);\n      if (entityId) {\n        state.anonymizationCount += 1;\n        entityHits.add(entityId);\n        return `<entity:${entityId}>`;\n      }\n    }\n    return match;\n  });\n  return { result, entityHits };\n}\n\nfunction transformText(\n  value: string,\n  options: PrivacyFilterOptions,\n  credentialValues: string[],\n  credentialPatterns: Array<{ label: string; pattern: RegExp }>,\n  state: InternalState,\n  collectedEntities: Set<string>,\n): string {\n  // Geo first so JSON `coords` blocks collapse before any later pass can see\n  // a stray decimal pair inside them.\n  const geoRedacted = redactGeo(value, state);\n  const credRedacted = redactCredentials(\n    geoRedacted,\n    credentialPatterns,\n    credentialValues,\n    state,\n  );\n  const piiRedacted = redactPii(credRedacted, state);\n  const { result, entityHits } = anonymizeHandles(piiRedacted, options, state);\n  for (const entityId of entityHits) collectedEntities.add(entityId);\n  return result;\n}\n\n/**\n * Recursively transform every string contained in `value` (objects, arrays,\n * and nested combinations). Returns the same shape with strings rewritten.\n */\nfunction transformDeep(\n  value: unknown,\n  options: PrivacyFilterOptions,\n  credentialValues: string[],\n  credentialPatterns: Array<{ label: string; pattern: RegExp }>,\n  state: InternalState,\n  collectedEntities: Set<string>,\n): unknown {\n  if (typeof value === \"string\") {\n    return transformText(\n      value,\n      options,\n      credentialValues,\n      credentialPatterns,\n      state,\n      collectedEntities,\n    );\n  }\n  if (Array.isArray(value)) {\n    return value.map((entry) =>\n      transformDeep(\n        entry,\n        options,\n        credentialValues,\n        credentialPatterns,\n        state,\n        collectedEntities,\n      ),\n    );\n  }\n  if (value && typeof value === \"object\") {\n    const out: Record<string, unknown> = {};\n    for (const [key, entry] of Object.entries(\n      value as Record<string, unknown>,\n    )) {\n      out[key] = transformDeep(\n        entry,\n        options,\n        credentialValues,\n        credentialPatterns,\n        state,\n        collectedEntities,\n      );\n    }\n    return out;\n  }\n  return value;\n}\n\n/**\n * Apply the privacy filter to a list of trajectories. Returns the filtered\n * list with credential references redacted and platform handles replaced by\n * opaque entity IDs. Trajectories whose anonymized entities are marked as\n * `private` are dropped wholesale.\n */\nexport function applyPrivacyFilter<T extends FilterableTrajectory>(\n  trajectories: T[],\n  options: PrivacyFilterOptions = {},\n): FilterResult<T> {\n  const credentialPatterns = [\n    ...DEFAULT_CREDENTIAL_PATTERNS,\n    ...(options.extraCredentialPatterns ?? []),\n  ];\n  const envKeys = options.envKeySnapshot ?? Object.keys(process.env);\n  const credentialValues = snapshotEnvCredentials(envKeys);\n\n  const dropped: Array<{ trajectoryId?: string; reason: string }> = [];\n  const filtered: T[] = [];\n  const state: InternalState = {\n    anonymizationCount: 0,\n    redactionCount: 0,\n  };\n\n  for (const trajectory of trajectories) {\n    const trajectoryEntities = new Set<string>();\n    const cloned = JSON.parse(JSON.stringify(trajectory)) as T;\n    const steps = cloned.steps ?? [];\n    for (const step of steps) {\n      for (const call of step.llmCalls ?? []) {\n        if (typeof call.systemPrompt === \"string\") {\n          call.systemPrompt = transformText(\n            call.systemPrompt,\n            options,\n            credentialValues,\n            credentialPatterns,\n            state,\n            trajectoryEntities,\n          );\n        }\n        if (typeof call.userPrompt === \"string\") {\n          call.userPrompt = transformText(\n            call.userPrompt,\n            options,\n            credentialValues,\n            credentialPatterns,\n            state,\n            trajectoryEntities,\n          );\n        }\n        if (typeof call.response === \"string\") {\n          call.response = transformText(\n            call.response,\n            options,\n            credentialValues,\n            credentialPatterns,\n            state,\n            trajectoryEntities,\n          );\n        }\n      }\n      for (const access of step.providerAccesses ?? []) {\n        if (access.data !== undefined) {\n          access.data = transformDeep(\n            access.data,\n            options,\n            credentialValues,\n            credentialPatterns,\n            state,\n            trajectoryEntities,\n          );\n        }\n      }\n    }\n\n    if (cloned.metadata && typeof cloned.metadata === \"object\") {\n      cloned.metadata = transformDeep(\n        cloned.metadata,\n        options,\n        credentialValues,\n        credentialPatterns,\n        state,\n        trajectoryEntities,\n      ) as Record<string, unknown>;\n    }\n\n    // Drop the whole trajectory if any participating entity is private.\n    const lookup = options.anonymizer?.getPrivacyLevel;\n    if (lookup) {\n      let isPrivate = false;\n      for (const entityId of trajectoryEntities) {\n        if (lookup(entityId) === \"private\") {\n          isPrivate = true;\n          break;\n        }\n      }\n      if (isPrivate) {\n        dropped.push({\n          trajectoryId: trajectory.trajectoryId,\n          reason: \"entity-private\",\n        });\n        continue;\n      }\n    }\n\n    filtered.push(cloned);\n  }\n\n  return {\n    trajectories: filtered,\n    dropped,\n    redactionCount: state.redactionCount,\n    anonymizationCount: state.anonymizationCount,\n  };\n}\n\n/**\n * Stable, dependency-free anonymizer: maps a `(platform, handle)` pair to a\n * 16-hex-character opaque id via `SHA-256(salt:platform:handle)`. The same\n * handle always resolves to the same id (for a given salt), so cross-message\n * references stay linkable in the exported corpus while the real handle is\n * gone. Returns the id for every handle (never `null`), so all `@mentions`\n * get anonymized.\n */\nexport function createHashAnonymizer(salt = \"\"): AnonymizerLookup {\n  return {\n    resolveEntityId(platform: string, handle: string): string {\n      return createHash(\"sha256\")\n        .update(`${salt}:${platform.toLowerCase()}:${handle.toLowerCase()}`)\n        .digest(\"hex\")\n        .slice(0, 16);\n    },\n  };\n}\n"],"mappings":"AAyBA,SAAS,kBAAkB;AAuD3B,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,iBAAiB;AAEvB,MAAM,8BAAyE;AAAA;AAAA;AAAA,EAG7E,EAAE,OAAO,iBAAiB,SAAS,iCAAiC;AAAA,EACpE,EAAE,OAAO,cAAc,SAAS,6BAA6B;AAAA,EAC7D;AAAA,IACE,OAAO;AAAA,IACP,SAAS;AAAA,EACX;AAAA,EACA;AAAA,IACE,OAAO;AAAA,IACP,SAAS;AAAA,EACX;AAAA,EACA;AAAA,IACE,OAAO;AAAA,IACP,SAAS;AAAA,EACX;AACF;AAQA,MAAM,oBAAoB;AAC1B,MAAM,oBAAoB;AAC1B,MAAM,sBAAsB;AAE5B,MAAM,gBAAgB;AAEtB,MAAM,kBACJ;AACF,MAAM,mBACJ;AACF,MAAM,YACJ;AAEF,MAAM,2BAAqC;AAAA;AAAA;AAAA;AAAA,EAIzC,IAAI;AAAA,IACF,OAAO,gDAAgD,eAAe,QACpE,OAAO,oBAAoB,gBAAgB,2BAC3C,OAAO,qCAAqC,SAAS;AAAA,IACvD;AAAA,EACF;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA,IAAI;AAAA,IACF,OAAO,4BAA4B,SAAS;AAAA,IAC5C;AAAA,EACF;AACF;AAEA,MAAM,yBAAmC;AAAA;AAAA;AAAA,EAGvC;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA;AACF;AAiBA,MAAM,kBAAkB;AAExB,MAAM,uBAAiC;AAAA;AAAA,EAErC;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA;AAAA;AAAA,EAGA;AACF;AAEA,SAAS,uBAAuB,SAA6B;AAG3D,QAAM,cAAc;AACpB,QAAM,MAAgB,CAAC;AACvB,aAAW,OAAO,SAAS;AACzB,QAAI,CAAC,YAAY,KAAK,GAAG,EAAG;AAC5B,UAAM,QAAQ,QAAQ,IAAI,GAAG;AAC7B,QAAI,OAAO,UAAU,SAAU;AAC/B,QAAI,MAAM,SAAS,EAAG;AACtB,QAAI,KAAK,KAAK;AAAA,EAChB;AACA,SAAO;AACT;AAOA,SAAS,kBACP,OACA,UACA,kBACA,OACQ;AACR,MAAI,MAAM;AACV,aAAW,EAAE,OAAO,QAAQ,KAAK,UAAU;AACzC,UAAM,IAAI,QAAQ,SAAS,MAAM;AAC/B,YAAM,kBAAkB;AACxB,aAAO,aAAa,KAAK;AAAA,IAC3B,CAAC;AAAA,EACH;AACA,aAAW,aAAa,kBAAkB;AACxC,QAAI,CAAC,UAAW;AAChB,UAAM,UAAU,UAAU,QAAQ,uBAAuB,MAAM;AAC/D,UAAM,KAAK,IAAI,OAAO,SAAS,GAAG;AAClC,UAAM,IAAI,QAAQ,IAAI,MAAM;AAC1B,YAAM,kBAAkB;AACxB,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,UAAU,OAAe,OAA8B;AAC9D,MAAI,MAAM;AACV,aAAW,WAAW,sBAAsB;AAC1C,UAAM,IAAI,QAAQ,SAAS,MAAM;AAC/B,YAAM,kBAAkB;AACxB,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,UAAU,OAAe,OAA8B;AAC9D,MAAI,MAAM;AAEV,QAAM,IAAI,QAAQ,eAAe,MAAM;AACrC,UAAM,kBAAkB;AACxB,WAAO;AAAA,EACT,CAAC;AACD,aAAW,WAAW,0BAA0B;AAC9C,UAAM,IAAI,QAAQ,SAAS,MAAM;AAC/B,YAAM,kBAAkB;AACxB,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AACA,aAAW,WAAW,wBAAwB;AAC5C,UAAM,IAAI,QAAQ,SAAS,MAAM;AAC/B,YAAM,kBAAkB;AACxB,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,iBACP,OACA,SACA,OAC6C;AAC7C,QAAM,YAAY,QAAQ,aAAa;AACvC,QAAM,aAAa,oBAAI,IAAY;AACnC,MAAI,CAAC,QAAQ,YAAY;AACvB,WAAO,EAAE,QAAQ,OAAO,WAAW;AAAA,EACrC;AAEA,QAAM,SAAS,MAAM,QAAQ,gBAAgB,CAAC,OAAO,WAAmB;AACtE,UAAM,WAAW,OAAO,WAAW,GAAG,IAAI,OAAO,MAAM,CAAC,IAAI;AAC5D,eAAW,YAAY,WAAW;AAChC,YAAM,WAAW,QAAQ,YAAY,gBAAgB,UAAU,QAAQ;AACvE,UAAI,UAAU;AACZ,cAAM,sBAAsB;AAC5B,mBAAW,IAAI,QAAQ;AACvB,eAAO,WAAW,QAAQ;AAAA,MAC5B;AAAA,IACF;AACA,WAAO;AAAA,EACT,CAAC;AACD,SAAO,EAAE,QAAQ,WAAW;AAC9B;AAEA,SAAS,cACP,OACA,SACA,kBACA,oBACA,OACA,mBACQ;AAGR,QAAM,cAAc,UAAU,OAAO,KAAK;AAC1C,QAAM,eAAe;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,QAAM,cAAc,UAAU,cAAc,KAAK;AACjD,QAAM,EAAE,QAAQ,WAAW,IAAI,iBAAiB,aAAa,SAAS,KAAK;AAC3E,aAAW,YAAY,WAAY,mBAAkB,IAAI,QAAQ;AACjE,SAAO;AACT;AAMA,SAAS,cACP,OACA,SACA,kBACA,oBACA,OACA,mBACS;AACT,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,MAAM;AAAA,MAAI,CAAC,UAChB;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,MAAI,SAAS,OAAO,UAAU,UAAU;AACtC,UAAM,MAA+B,CAAC;AACtC,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO;AAAA,MAChC;AAAA,IACF,GAAG;AACD,UAAI,GAAG,IAAI;AAAA,QACT;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAQO,SAAS,mBACd,cACA,UAAgC,CAAC,GAChB;AACjB,QAAM,qBAAqB;AAAA,IACzB,GAAG;AAAA,IACH,GAAI,QAAQ,2BAA2B,CAAC;AAAA,EAC1C;AACA,QAAM,UAAU,QAAQ,kBAAkB,OAAO,KAAK,QAAQ,GAAG;AACjE,QAAM,mBAAmB,uBAAuB,OAAO;AAEvD,QAAM,UAA4D,CAAC;AACnE,QAAM,WAAgB,CAAC;AACvB,QAAM,QAAuB;AAAA,IAC3B,oBAAoB;AAAA,IACpB,gBAAgB;AAAA,EAClB;AAEA,aAAW,cAAc,cAAc;AACrC,UAAM,qBAAqB,oBAAI,IAAY;AAC3C,UAAM,SAAS,KAAK,MAAM,KAAK,UAAU,UAAU,CAAC;AACpD,UAAM,QAAQ,OAAO,SAAS,CAAC;AAC/B,eAAW,QAAQ,OAAO;AACxB,iBAAW,QAAQ,KAAK,YAAY,CAAC,GAAG;AACtC,YAAI,OAAO,KAAK,iBAAiB,UAAU;AACzC,eAAK,eAAe;AAAA,YAClB,KAAK;AAAA,YACL;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA,YAAI,OAAO,KAAK,eAAe,UAAU;AACvC,eAAK,aAAa;AAAA,YAChB,KAAK;AAAA,YACL;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,QACF;AACA,YAAI,OAAO,KAAK,aAAa,UAAU;AACrC,eAAK,WAAW;AAAA,YACd,KAAK;AAAA,YACL;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF;AACA,iBAAW,UAAU,KAAK,oBAAoB,CAAC,GAAG;AAChD,YAAI,OAAO,SAAS,QAAW;AAC7B,iBAAO,OAAO;AAAA,YACZ,OAAO;AAAA,YACP;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,OAAO,YAAY,OAAO,OAAO,aAAa,UAAU;AAC1D,aAAO,WAAW;AAAA,QAChB,OAAO;AAAA,QACP;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAGA,UAAM,SAAS,QAAQ,YAAY;AACnC,QAAI,QAAQ;AACV,UAAI,YAAY;AAChB,iBAAW,YAAY,oBAAoB;AACzC,YAAI,OAAO,QAAQ,MAAM,WAAW;AAClC,sBAAY;AACZ;AAAA,QACF;AAAA,MACF;AACA,UAAI,WAAW;AACb,gBAAQ,KAAK;AAAA,UACX,cAAc,WAAW;AAAA,UACzB,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAAA,IACF;AAEA,aAAS,KAAK,MAAM;AAAA,EACtB;AAEA,SAAO;AAAA,IACL,cAAc;AAAA,IACd;AAAA,IACA,gBAAgB,MAAM;AAAA,IACtB,oBAAoB,MAAM;AAAA,EAC5B;AACF;AAUO,SAAS,qBAAqB,OAAO,IAAsB;AAChE,SAAO;AAAA,IACL,gBAAgB,UAAkB,QAAwB;AACxD,aAAO,WAAW,QAAQ,EACvB,OAAO,GAAG,IAAI,IAAI,SAAS,YAAY,CAAC,IAAI,OAAO,YAAY,CAAC,EAAE,EAClE,OAAO,KAAK,EACZ,MAAM,GAAG,EAAE;AAAA,IAChB;AAAA,EACF;AACF;","names":[]}

package/dist/core/promotion-gate.d.ts

@@ -0,0 +1,117 @@
+/**
+ * A/B promotion gate for optimized prompt artifacts.
+ *
+ * Native MIPRO/GEPA/bootstrap-fewshot runs in `backends/native.ts` produce a
+ * candidate prompt for a task. Without a gate, that candidate is written as the
+ * `current` artifact unconditionally, which lets noisy single-run scores
+ * silently regress production prompts.
+ *
+ * This module evaluates a candidate against its incumbent (the prompt currently
+ * loaded by `OptimizedPromptService` — or the baseline when no artifact exists
+ * yet) on a held-out trajectory replay set, and only promotes when the
+ * candidate's score exceeds the incumbent score by more than the expected
+ * scoring noise (default: 1.5× the standard deviation of incumbent scores
+ * across reseeded scoring runs).
+ *
+ * Inputs and outputs are pure JS objects — no filesystem, no service lookups.
+ * The orchestrator passes incumbent text + dataset in, gets a structured
+ * decision back, and is responsible for persistence (promote → write artifact;
+ * reject → write `candidate_rejected_<timestamp>.json`).
+ */
+import type { OptimizationExample, PromptScorer } from "../optimizers/types.js";
+/**
+ * Default noise threshold. A candidate must beat the incumbent's mean by
+ * `noiseThreshold × stddev(incumbent)` to be promoted. 1.5× is the same
+ * multiplier the MIPRO paper uses for its variance-aware acceptance test.
+ */
+export declare const DEFAULT_NOISE_THRESHOLD = 1.5;
+/**
+ * Default number of times the incumbent is re-scored to estimate scoring noise.
+ * Each pass uses a fresh subsample (when `reseedSubsample` is set) so the
+ * resulting stddev captures both sampling jitter and scorer non-determinism.
+ */
+export declare const DEFAULT_INCUMBENT_RESEEDS = 3;
+export interface PromotionGateOptions {
+    /**
+     * Multiplier applied to the incumbent stddev. Default `1.5`.
+     * Set to 0 to promote on any positive delta (not recommended).
+     */
+    noiseThreshold?: number;
+    /**
+     * Reseeded incumbent scoring passes. Default `3`. Each pass produces an
+     * independent score on a fresh subsample (when `scoringSubset` is set), or on
+     * the full dataset otherwise. More passes → tighter stddev estimate at the
+     * cost of more model calls.
+     */
+    incumbentReseeds?: number;
+    /**
+     * Cap on examples scored per pass. When set, each incumbent reseed and the
+     * single candidate pass each draw their own subsample. Defaults to all rows.
+     */
+    scoringSubset?: number;
+    /**
+     * Deterministic RNG override (tests). Defaults to `Math.random`.
+     */
+    rng?: () => number;
+    /**
+     * When true, the candidate is scored on a freshly subsampled held-out set
+     * (independent of the incumbent reseed samples). Default: true. Set false to
+     * reuse the union of all incumbent samples.
+     */
+    reseedCandidate?: boolean;
+}
+export interface PromotionGateInput {
+    /** Prompt currently in production for this task. */
+    incumbentPrompt: string;
+    /** Candidate prompt produced by the optimizer for this task. */
+    candidatePrompt: string;
+    /** Replay dataset — same shape the optimizers consume. */
+    dataset: OptimizationExample[];
+    /** Scorer used for both incumbent and candidate. */
+    scorer: PromptScorer;
+    options?: PromotionGateOptions;
+}
+export interface PromotionDecision {
+    /** `true` when the candidate should replace the incumbent. */
+    promote: boolean;
+    /** Mean score of the incumbent across reseeded passes. */
+    incumbentMeanScore: number;
+    /** Stddev of incumbent scores across reseeded passes. */
+    incumbentStdDev: number;
+    /** Score of the candidate on its (re)sampled held-out set. */
+    candidateScore: number;
+    /** `candidateScore - incumbentMeanScore`. Positive = candidate better. */
+    delta: number;
+    /** `noiseThreshold * incumbentStdDev`. Candidate must beat this. */
+    promotionMargin: number;
+    /** Multiplier used (`options.noiseThreshold ?? DEFAULT_NOISE_THRESHOLD`). */
+    noiseThreshold: number;
+    /** Number of reseeded incumbent passes actually run. */
+    incumbentReseeds: number;
+    /** Number of rows scored per pass. */
+    examplesPerPass: number;
+    /** Plain-english reason describing why the gate accepted or rejected. */
+    reason: string;
+    /** Raw per-pass incumbent scores, oldest first. */
+    incumbentScores: number[];
+}
+/**
+ * Evaluate whether a candidate prompt should be promoted over the incumbent.
+ *
+ * Algorithm:
+ *   1. Score the incumbent `incumbentReseeds` times on independently subsampled
+ *      held-out sets (or the full dataset when `scoringSubset` is unset).
+ *   2. Compute the mean and population stddev of the resulting scores.
+ *   3. Score the candidate once on a fresh subsample (or the full dataset).
+ *   4. Promote only when `candidateScore > incumbentMean + noiseThreshold * stddev`.
+ *
+ * Edge cases:
+ *   - Empty dataset → never promote (delta=0, candidate cannot demonstrate
+ *     improvement).
+ *   - `incumbentReseeds < 1` → rejected outright; we need at least one
+ *     measurement to gate against.
+ *   - When all incumbent passes return the exact same score, stddev=0 and the
+ *     candidate only needs to strictly exceed the incumbent mean.
+ */
+export declare function evaluatePromotion(input: PromotionGateInput): Promise<PromotionDecision>;
+//# sourceMappingURL=promotion-gate.d.ts.map

package/dist/core/promotion-gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"promotion-gate.d.ts","sourceRoot":"","sources":["../../src/core/promotion-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAE3C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,oDAAoD;IACpD,eAAe,EAAE,MAAM,CAAC;IACxB,gEAAgE;IAChE,eAAe,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,oDAAoD;IACpD,MAAM,EAAE,YAAY,CAAC;IACrB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC;AAED,MAAM,WAAW,iBAAiB;IAChC,8DAA8D;IAC9D,OAAO,EAAE,OAAO,CAAC;IACjB,0DAA0D;IAC1D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,eAAe,EAAE,MAAM,CAAC;IACxB,8DAA8D;IAC9D,cAAc,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,oEAAoE;IACpE,eAAe,EAAE,MAAM,CAAC;IACxB,6EAA6E;IAC7E,cAAc,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IACzB,sCAAsC;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,iBAAiB,CAAC,CA+E5B"}

package/dist/core/promotion-gate.js

@@ -0,0 +1,85 @@
+import { subsample } from "../optimizers/scoring.js";
+const DEFAULT_NOISE_THRESHOLD = 1.5;
+const DEFAULT_INCUMBENT_RESEEDS = 3;
+async function evaluatePromotion(input) {
+  const noiseThreshold = input.options?.noiseThreshold ?? DEFAULT_NOISE_THRESHOLD;
+  const incumbentReseeds = Math.max(
+    1,
+    input.options?.incumbentReseeds ?? DEFAULT_INCUMBENT_RESEEDS
+  );
+  const rng = input.options?.rng ?? Math.random;
+  const reseedCandidate = input.options?.reseedCandidate ?? true;
+  const examplesPerPass = typeof input.options?.scoringSubset === "number" ? Math.min(input.options.scoringSubset, input.dataset.length) : input.dataset.length;
+  if (input.dataset.length === 0 || examplesPerPass === 0) {
+    return {
+      promote: false,
+      incumbentMeanScore: 0,
+      incumbentStdDev: 0,
+      candidateScore: 0,
+      delta: 0,
+      promotionMargin: 0,
+      noiseThreshold,
+      incumbentReseeds: 0,
+      examplesPerPass: 0,
+      reason: "dataset is empty; cannot evaluate promotion",
+      incumbentScores: []
+    };
+  }
+  const incumbentScores = [];
+  let lastIncumbentSample = null;
+  for (let i = 0; i < incumbentReseeds; i += 1) {
+    const sample = drawSample(input.dataset, examplesPerPass, rng);
+    const score = await input.scorer(input.incumbentPrompt, sample);
+    incumbentScores.push(score);
+    lastIncumbentSample = sample;
+  }
+  const incumbentMean = mean(incumbentScores);
+  const incumbentStd = populationStdDev(incumbentScores, incumbentMean);
+  const candidateSample = reseedCandidate || !lastIncumbentSample ? drawSample(input.dataset, examplesPerPass, rng) : lastIncumbentSample;
+  const candidateScore = await input.scorer(
+    input.candidatePrompt,
+    candidateSample
+  );
+  const promotionMargin = noiseThreshold * incumbentStd;
+  const delta = candidateScore - incumbentMean;
+  const promote = delta > promotionMargin;
+  const reason = promote ? `candidate beats incumbent by ${delta.toFixed(4)} > margin ${promotionMargin.toFixed(4)} (${noiseThreshold}\xD7 stddev=${incumbentStd.toFixed(4)})` : delta <= 0 ? `candidate did not improve over incumbent (delta=${delta.toFixed(4)})` : `candidate improvement ${delta.toFixed(4)} did not exceed noise margin ${promotionMargin.toFixed(4)} (${noiseThreshold}\xD7 stddev=${incumbentStd.toFixed(4)})`;
+  return {
+    promote,
+    incumbentMeanScore: incumbentMean,
+    incumbentStdDev: incumbentStd,
+    candidateScore,
+    delta,
+    promotionMargin,
+    noiseThreshold,
+    incumbentReseeds,
+    examplesPerPass,
+    reason,
+    incumbentScores
+  };
+}
+function drawSample(dataset, count, rng) {
+  if (count >= dataset.length) return [...dataset];
+  return subsample(dataset, count, rng);
+}
+function mean(values) {
+  if (values.length === 0) return 0;
+  let total = 0;
+  for (const v of values) total += v;
+  return total / values.length;
+}
+function populationStdDev(values, precomputedMean) {
+  if (values.length === 0) return 0;
+  let total = 0;
+  for (const v of values) {
+    const d = v - precomputedMean;
+    total += d * d;
+  }
+  return Math.sqrt(total / values.length);
+}
+export {
+  DEFAULT_INCUMBENT_RESEEDS,
+  DEFAULT_NOISE_THRESHOLD,
+  evaluatePromotion
+};
+//# sourceMappingURL=promotion-gate.js.map

package/dist/core/promotion-gate.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/core/promotion-gate.ts"],"sourcesContent":["/**\n * A/B promotion gate for optimized prompt artifacts.\n *\n * Native MIPRO/GEPA/bootstrap-fewshot runs in `backends/native.ts` produce a\n * candidate prompt for a task. Without a gate, that candidate is written as the\n * `current` artifact unconditionally, which lets noisy single-run scores\n * silently regress production prompts.\n *\n * This module evaluates a candidate against its incumbent (the prompt currently\n * loaded by `OptimizedPromptService` — or the baseline when no artifact exists\n * yet) on a held-out trajectory replay set, and only promotes when the\n * candidate's score exceeds the incumbent score by more than the expected\n * scoring noise (default: 1.5× the standard deviation of incumbent scores\n * across reseeded scoring runs).\n *\n * Inputs and outputs are pure JS objects — no filesystem, no service lookups.\n * The orchestrator passes incumbent text + dataset in, gets a structured\n * decision back, and is responsible for persistence (promote → write artifact;\n * reject → write `candidate_rejected_<timestamp>.json`).\n */\n\nimport { subsample } from \"../optimizers/scoring.js\";\nimport type { OptimizationExample, PromptScorer } from \"../optimizers/types.js\";\n\n/**\n * Default noise threshold. A candidate must beat the incumbent's mean by\n * `noiseThreshold × stddev(incumbent)` to be promoted. 1.5× is the same\n * multiplier the MIPRO paper uses for its variance-aware acceptance test.\n */\nexport const DEFAULT_NOISE_THRESHOLD = 1.5;\n\n/**\n * Default number of times the incumbent is re-scored to estimate scoring noise.\n * Each pass uses a fresh subsample (when `reseedSubsample` is set) so the\n * resulting stddev captures both sampling jitter and scorer non-determinism.\n */\nexport const DEFAULT_INCUMBENT_RESEEDS = 3;\n\nexport interface PromotionGateOptions {\n  /**\n   * Multiplier applied to the incumbent stddev. Default `1.5`.\n   * Set to 0 to promote on any positive delta (not recommended).\n   */\n  noiseThreshold?: number;\n  /**\n   * Reseeded incumbent scoring passes. Default `3`. Each pass produces an\n   * independent score on a fresh subsample (when `scoringSubset` is set), or on\n   * the full dataset otherwise. More passes → tighter stddev estimate at the\n   * cost of more model calls.\n   */\n  incumbentReseeds?: number;\n  /**\n   * Cap on examples scored per pass. When set, each incumbent reseed and the\n   * single candidate pass each draw their own subsample. Defaults to all rows.\n   */\n  scoringSubset?: number;\n  /**\n   * Deterministic RNG override (tests). Defaults to `Math.random`.\n   */\n  rng?: () => number;\n  /**\n   * When true, the candidate is scored on a freshly subsampled held-out set\n   * (independent of the incumbent reseed samples). Default: true. Set false to\n   * reuse the union of all incumbent samples.\n   */\n  reseedCandidate?: boolean;\n}\n\nexport interface PromotionGateInput {\n  /** Prompt currently in production for this task. */\n  incumbentPrompt: string;\n  /** Candidate prompt produced by the optimizer for this task. */\n  candidatePrompt: string;\n  /** Replay dataset — same shape the optimizers consume. */\n  dataset: OptimizationExample[];\n  /** Scorer used for both incumbent and candidate. */\n  scorer: PromptScorer;\n  options?: PromotionGateOptions;\n}\n\nexport interface PromotionDecision {\n  /** `true` when the candidate should replace the incumbent. */\n  promote: boolean;\n  /** Mean score of the incumbent across reseeded passes. */\n  incumbentMeanScore: number;\n  /** Stddev of incumbent scores across reseeded passes. */\n  incumbentStdDev: number;\n  /** Score of the candidate on its (re)sampled held-out set. */\n  candidateScore: number;\n  /** `candidateScore - incumbentMeanScore`. Positive = candidate better. */\n  delta: number;\n  /** `noiseThreshold * incumbentStdDev`. Candidate must beat this. */\n  promotionMargin: number;\n  /** Multiplier used (`options.noiseThreshold ?? DEFAULT_NOISE_THRESHOLD`). */\n  noiseThreshold: number;\n  /** Number of reseeded incumbent passes actually run. */\n  incumbentReseeds: number;\n  /** Number of rows scored per pass. */\n  examplesPerPass: number;\n  /** Plain-english reason describing why the gate accepted or rejected. */\n  reason: string;\n  /** Raw per-pass incumbent scores, oldest first. */\n  incumbentScores: number[];\n}\n\n/**\n * Evaluate whether a candidate prompt should be promoted over the incumbent.\n *\n * Algorithm:\n *   1. Score the incumbent `incumbentReseeds` times on independently subsampled\n *      held-out sets (or the full dataset when `scoringSubset` is unset).\n *   2. Compute the mean and population stddev of the resulting scores.\n *   3. Score the candidate once on a fresh subsample (or the full dataset).\n *   4. Promote only when `candidateScore > incumbentMean + noiseThreshold * stddev`.\n *\n * Edge cases:\n *   - Empty dataset → never promote (delta=0, candidate cannot demonstrate\n *     improvement).\n *   - `incumbentReseeds < 1` → rejected outright; we need at least one\n *     measurement to gate against.\n *   - When all incumbent passes return the exact same score, stddev=0 and the\n *     candidate only needs to strictly exceed the incumbent mean.\n */\nexport async function evaluatePromotion(\n  input: PromotionGateInput,\n): Promise<PromotionDecision> {\n  const noiseThreshold =\n    input.options?.noiseThreshold ?? DEFAULT_NOISE_THRESHOLD;\n  const incumbentReseeds = Math.max(\n    1,\n    input.options?.incumbentReseeds ?? DEFAULT_INCUMBENT_RESEEDS,\n  );\n  const rng = input.options?.rng ?? Math.random;\n  const reseedCandidate = input.options?.reseedCandidate ?? true;\n  const examplesPerPass =\n    typeof input.options?.scoringSubset === \"number\"\n      ? Math.min(input.options.scoringSubset, input.dataset.length)\n      : input.dataset.length;\n\n  if (input.dataset.length === 0 || examplesPerPass === 0) {\n    return {\n      promote: false,\n      incumbentMeanScore: 0,\n      incumbentStdDev: 0,\n      candidateScore: 0,\n      delta: 0,\n      promotionMargin: 0,\n      noiseThreshold,\n      incumbentReseeds: 0,\n      examplesPerPass: 0,\n      reason: \"dataset is empty; cannot evaluate promotion\",\n      incumbentScores: [],\n    };\n  }\n\n  const incumbentScores: number[] = [];\n  let lastIncumbentSample: OptimizationExample[] | null = null;\n  for (let i = 0; i < incumbentReseeds; i += 1) {\n    const sample = drawSample(input.dataset, examplesPerPass, rng);\n    const score = await input.scorer(input.incumbentPrompt, sample);\n    incumbentScores.push(score);\n    lastIncumbentSample = sample;\n  }\n\n  const incumbentMean = mean(incumbentScores);\n  const incumbentStd = populationStdDev(incumbentScores, incumbentMean);\n\n  // When `reseedCandidate` is false the candidate is scored on the exact same\n  // examples the final incumbent pass saw — useful for direct A/B comparison\n  // without sampling jitter. Defaults to true (fresh subsample) so the\n  // candidate score is independent of which rows happened to land in the last\n  // incumbent reseed.\n  const candidateSample =\n    reseedCandidate || !lastIncumbentSample\n      ? drawSample(input.dataset, examplesPerPass, rng)\n      : lastIncumbentSample;\n  const candidateScore = await input.scorer(\n    input.candidatePrompt,\n    candidateSample,\n  );\n\n  const promotionMargin = noiseThreshold * incumbentStd;\n  const delta = candidateScore - incumbentMean;\n  const promote = delta > promotionMargin;\n\n  const reason = promote\n    ? `candidate beats incumbent by ${delta.toFixed(4)} > margin ${promotionMargin.toFixed(4)} (${noiseThreshold}× stddev=${incumbentStd.toFixed(4)})`\n    : delta <= 0\n      ? `candidate did not improve over incumbent (delta=${delta.toFixed(4)})`\n      : `candidate improvement ${delta.toFixed(4)} did not exceed noise margin ${promotionMargin.toFixed(4)} (${noiseThreshold}× stddev=${incumbentStd.toFixed(4)})`;\n\n  return {\n    promote,\n    incumbentMeanScore: incumbentMean,\n    incumbentStdDev: incumbentStd,\n    candidateScore,\n    delta,\n    promotionMargin,\n    noiseThreshold,\n    incumbentReseeds,\n    examplesPerPass,\n    reason,\n    incumbentScores,\n  };\n}\n\nfunction drawSample(\n  dataset: OptimizationExample[],\n  count: number,\n  rng: () => number,\n): OptimizationExample[] {\n  if (count >= dataset.length) return [...dataset];\n  return subsample(dataset, count, rng);\n}\n\nfunction mean(values: number[]): number {\n  if (values.length === 0) return 0;\n  let total = 0;\n  for (const v of values) total += v;\n  return total / values.length;\n}\n\n/**\n * Population stddev (divide by N, not N-1). With small N (default 3 reseeds)\n * the sample stddev estimator inflates noise enough that even slightly better\n * candidates get rejected; the gate is already conservative through the\n * `noiseThreshold` multiplier, so we use the population form here.\n */\nfunction populationStdDev(values: number[], precomputedMean: number): number {\n  if (values.length === 0) return 0;\n  let total = 0;\n  for (const v of values) {\n    const d = v - precomputedMean;\n    total += d * d;\n  }\n  return Math.sqrt(total / values.length);\n}\n"],"mappings":"AAqBA,SAAS,iBAAiB;AAQnB,MAAM,0BAA0B;AAOhC,MAAM,4BAA4B;AAuFzC,eAAsB,kBACpB,OAC4B;AAC5B,QAAM,iBACJ,MAAM,SAAS,kBAAkB;AACnC,QAAM,mBAAmB,KAAK;AAAA,IAC5B;AAAA,IACA,MAAM,SAAS,oBAAoB;AAAA,EACrC;AACA,QAAM,MAAM,MAAM,SAAS,OAAO,KAAK;AACvC,QAAM,kBAAkB,MAAM,SAAS,mBAAmB;AAC1D,QAAM,kBACJ,OAAO,MAAM,SAAS,kBAAkB,WACpC,KAAK,IAAI,MAAM,QAAQ,eAAe,MAAM,QAAQ,MAAM,IAC1D,MAAM,QAAQ;AAEpB,MAAI,MAAM,QAAQ,WAAW,KAAK,oBAAoB,GAAG;AACvD,WAAO;AAAA,MACL,SAAS;AAAA,MACT,oBAAoB;AAAA,MACpB,iBAAiB;AAAA,MACjB,gBAAgB;AAAA,MAChB,OAAO;AAAA,MACP,iBAAiB;AAAA,MACjB;AAAA,MACA,kBAAkB;AAAA,MAClB,iBAAiB;AAAA,MACjB,QAAQ;AAAA,MACR,iBAAiB,CAAC;AAAA,IACpB;AAAA,EACF;AAEA,QAAM,kBAA4B,CAAC;AACnC,MAAI,sBAAoD;AACxD,WAAS,IAAI,GAAG,IAAI,kBAAkB,KAAK,GAAG;AAC5C,UAAM,SAAS,WAAW,MAAM,SAAS,iBAAiB,GAAG;AAC7D,UAAM,QAAQ,MAAM,MAAM,OAAO,MAAM,iBAAiB,MAAM;AAC9D,oBAAgB,KAAK,KAAK;AAC1B,0BAAsB;AAAA,EACxB;AAEA,QAAM,gBAAgB,KAAK,eAAe;AAC1C,QAAM,eAAe,iBAAiB,iBAAiB,aAAa;AAOpE,QAAM,kBACJ,mBAAmB,CAAC,sBAChB,WAAW,MAAM,SAAS,iBAAiB,GAAG,IAC9C;AACN,QAAM,iBAAiB,MAAM,MAAM;AAAA,IACjC,MAAM;AAAA,IACN;AAAA,EACF;AAEA,QAAM,kBAAkB,iBAAiB;AACzC,QAAM,QAAQ,iBAAiB;AAC/B,QAAM,UAAU,QAAQ;AAExB,QAAM,SAAS,UACX,gCAAgC,MAAM,QAAQ,CAAC,CAAC,aAAa,gBAAgB,QAAQ,CAAC,CAAC,KAAK,cAAc,eAAY,aAAa,QAAQ,CAAC,CAAC,MAC7I,SAAS,IACP,mDAAmD,MAAM,QAAQ,CAAC,CAAC,MACnE,yBAAyB,MAAM,QAAQ,CAAC,CAAC,gCAAgC,gBAAgB,QAAQ,CAAC,CAAC,KAAK,cAAc,eAAY,aAAa,QAAQ,CAAC,CAAC;AAE/J,SAAO;AAAA,IACL;AAAA,IACA,oBAAoB;AAAA,IACpB,iBAAiB;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,WACP,SACA,OACA,KACuB;AACvB,MAAI,SAAS,QAAQ,OAAQ,QAAO,CAAC,GAAG,OAAO;AAC/C,SAAO,UAAU,SAAS,OAAO,GAAG;AACtC;AAEA,SAAS,KAAK,QAA0B;AACtC,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,QAAQ;AACZ,aAAW,KAAK,OAAQ,UAAS;AACjC,SAAO,QAAQ,OAAO;AACxB;AAQA,SAAS,iBAAiB,QAAkB,iBAAiC;AAC3E,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,QAAQ;AACZ,aAAW,KAAK,QAAQ;AACtB,UAAM,IAAI,IAAI;AACd,aAAS,IAAI;AAAA,EACf;AACA,SAAO,KAAK,KAAK,QAAQ,OAAO,MAAM;AACxC;","names":[]}