@pwguler/pi-pengepul-provider 0.5.0 → 0.5.1

package/README.md

@@ -58,6 +58,10 @@ prefixed `pengepul/<id>`. To try it without installing, use
   relay already serves — takes them from the same family's previous minor, so
   it does not sit at `high` until the next pi catalog update. Nothing to
   configure: `thinkingLevelMap` is resolved per id here.
+- Publishes Anthropic's cache lifetimes (5 minutes, 1 hour) on the Messages
+  wire, so pi's cache warmer can keep a Claude conversation's entry alive
+  instead of re-billing its whole prefix after five idle minutes. See
+  [Prompt cache](#prompt-cache).
 - Caches the catalog in pi's model store, so startup does not wait on the
   network and a briefly absent relay is covered. The cached models are
   re-pointed at the relay base configured now, so moving the relay does not
@@ -107,6 +111,47 @@ to every model of the provider, which collapses the two wires onto one URL —
 Anthropic Messages traffic would be sent to `/v1` and Chat Completions traffic
 to `/`.
 
+## Prompt cache
+
+The upstream prompt cache is one entry per account and lives five minutes by
+default, so a conversation that waits longer than that pays for its whole prefix
+again — on a 700k-token Opus session, dollars per return, logged by the relay as
+a `cache_write` with a near-zero `cache_read`.
+
+This provider declares Anthropic's lifetimes (5 minutes, 1 hour) on every
+Messages-wire model, which is what lets pi keep an entry alive at all: pi
+refreshes an entry by re-sending the prompt with a one-token cap before it
+expires, and it only warms a model whose lifetime it can resolve. Two pi
+settings decide when that happens, and neither covers an idle gap out of the
+box:
+
+| Setting | Where | Effect |
+|---|---|---|
+| `cacheWarming: "idle"` | `~/.pi/agent/settings.json` | Warms between agent runs. The default, `"streaming"`, stops as soon as a run settles — exactly when a five-minute entry is about to expire. |
+| `env: { "PI_CACHE_RETENTION": "long" }` | the `pengepul` credential in `auth.json` | Asks for the 1-hour tier instead of the 5-minute one. The relay forwards the extended TTL to Anthropic. |
+
+```json
+{
+  "pengepul": {
+    "type": "api_key",
+    "key": "sk-local-...",
+    "baseUrl": "http://127.0.0.1:8317",
+    "env": { "PI_CACHE_RETENTION": "long" }
+  }
+}
+```
+
+Idle warming covers gaps up to 30 minutes, pi's own safety limit, and pi only
+triggers it when the expected saving clears a threshold of its own — a small
+prompt is left to expire rather than refreshed. The long tier is what carries a
+longer break. Both are cheap next to a miss: a refresh is one cache read plus a
+one-token completion, while the long tier only raises the write premium, on the
+few hundred tokens a turn actually adds.
+
+`/session` reports the mode, the refresh cost, and the miss penalty it is
+weighing, and `showCacheMissNotices` in `settings.json` prints each miss with
+its re-billed token count.
+
 ## Upgrading from 0.4.0 or earlier
 
 This release is breaking: the relay's own `~/.pengepul/config.yaml` is no longer

package/extensions/models.ts

@@ -522,8 +522,22 @@ export function modelsFromApiResponse(
 
 import type { Api, Model } from "@earendil-works/pi-ai"
 
+/**
+ * Prompt cache lifetime in seconds for each retention tier a request can ask
+ * for, the shape pi reads off a model to decide when to refresh its cache
+ * entry. Declared here because pi-ai 0.85.1's `Model` has no `promptCache`;
+ * a host that predates the field ignores it.
+ */
+export interface PromptCacheTiers {
+  short: number
+  long: number
+}
+
 /** The pi-ai model shapes this provider serves: one per dialect the relay speaks. */
-export type PengepulModelEntry = Model<"anthropic-messages"> | Model<"openai-completions">
+export type PengepulModelEntry = (
+  | Model<"anthropic-messages">
+  | Model<"openai-completions">
+) & { promptCache?: PromptCacheTiers }
 
 /** The provider id every pengepul model is stamped with. */
 export const PENGEPUL_PROVIDER_ID = "pengepul"
@@ -562,18 +576,17 @@ function sharedModelFields(model: PengepulModel, relayBase: string) {
  * openai-completions carries `prompt_cache_key: <sessionId>` (and
  * `prompt_cache_retention: "24h"`) under PI_CACHE_RETENTION=long, so the body
  * field alone would name the conversation; anthropic-messages carries no
- * `prompt_cache_key` at all, and pi-ai hardcodes `x-session-affinity` there,
- * which this relay does not read. Messages traffic therefore rests entirely on
- * the relay's prefix fallback until a pi release honours
- * `sessionAffinityFormat` on that dialect.
+ * `prompt_cache_key` at all, and pi-ai hardcodes `x-session-affinity` there up
+ * to 0.85.1, which this relay does not read. Messages traffic therefore rests
+ * on the relay's prefix fallback until 0.87, where `sessionAffinityFormat`
+ * starts being read on that dialect too and the same `openrouter` value lands
+ * as `x-session-id`.
  *
- * The two dialects do not land at the same time. openai-completions honors
- * `sessionAffinityFormat` in every released pi. anthropic-messages only reads it
- * from the unreleased change on pi main (commit bbb61e34a), which is why the
- * field is re-declared below rather than taken from `AnthropicMessagesCompat`:
- * against pi-ai 0.85.1 the pin is inert, and the cost of an ignored header is
- * zero. Pin now rather than later — the cost of forgetting is silently
- * re-billed Claude prefixes.
+ * The field is re-declared below rather than taken from
+ * `AnthropicMessagesCompat` so the pin does not depend on the host's pi-ai
+ * version: 0.85.1 ignores a value it never types, and the cost of an ignored
+ * header is zero. Pin now rather than later — the cost of forgetting is
+ * silently re-billed Claude prefixes.
  */
 function affinityPin() {
   return {
@@ -582,6 +595,32 @@ function affinityPin() {
   }
 }
 
+/**
+ * Anthropic's prompt cache lifetimes: five minutes by default, one hour when
+ * the request asks for the extended TTL. Copied per model rather than shared,
+ * so nothing downstream can mutate one model's lifetimes into another's.
+ *
+ * pi reads `model.promptCache[tier]` to know when the entry a request wrote
+ * expires, and skips warming a model whose lifetime it cannot resolve
+ * (`cache-warmer.js` `getPromptCacheTtlMs`). No pi catalog answers for a relay
+ * id, so a Claude model served here would otherwise never be warmed: any idle
+ * past the TTL — five minutes on the default tier — re-bills the whole prefix
+ * at the write rate, which for a 700k-token Opus conversation is dollars per
+ * miss.
+ *
+ * Only the Messages wire gets them. Those ids are the ones the relay forwards
+ * to Anthropic's own API, whose lifetimes these are; the Chat Completions wire
+ * serves aggregated vendors (commandcode, openrouter) through upstreams whose
+ * cache lifetimes this provider has no measurement for, and pi does not warm a
+ * lifetime it cannot resolve.
+ *
+ * Measured, not assumed: a `cache_control` ttl of `1h` on this wire comes back
+ * with the whole prefix in `usage.cache_creation.ephemeral_1h_input_tokens`,
+ * so the relay passes the extended TTL through and `PI_CACHE_RETENTION=long`
+ * buys the hour declared here.
+ */
+const ANTHROPIC_PROMPT_CACHE: PromptCacheTiers = { short: 300, long: 3600 }
+
 /**
  * Anthropic compat as pi-ai 0.85.1 types it, plus the affinity format a later
  * pi reads. Declared here so the pin does not depend on the host's pi-ai
@@ -633,6 +672,10 @@ export function toPengepulModels(
         // The 1h cache TTL is a Messages-dialect feature too: `cache_control.ttl`
         // has nowhere to go on the Chat Completions wire. Reasoning is not part
         // of it — a non-reasoning Claude model caches the same way.
+        //
+        // The lifetimes pi's cache warmer schedules its refreshes against, on
+        // this wire only, for the reason just given.
+        promptCache: { ...ANTHROPIC_PROMPT_CACHE },
         compat: {
           ...affinityPin(),
           ...(adaptive ? { forceAdaptiveThinking: true as const } : {}),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pwguler/pi-pengepul-provider",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "type": "module",
   "description": "pi custom provider for pengepul, a relay that pools your Claude/Codex subscriptions. Key and relay base live in auth.json; both native wires (Anthropic Messages, OpenAI Chat Completions) come from one catalog.",
   "license": "MIT",