@spendgraph/sdk 0.1.0 → 0.2.0

package/dist/track/track.js

@@ -0,0 +1,513 @@
+import { Client, SpendgraphError } from "../core/client/index.js";
+/** How long a flush may block waiting out a rate limit before giving up. */
+const MAX_RETRY_WAIT_MS = 2_000;
+/**
+ * Last-chance flush when the process winds down, shared by every meter.
+ *
+ * `beforeExit` is the right hook — unlike `exit` it permits async work, so the
+ * final send can actually complete. But it used to be registered per instance,
+ * so an app holding more than ten meters tripped Node's
+ * MaxListenersExceededWarning and kept every one of them alive forever.
+ *
+ * One listener, and meters are held weakly so a discarded one can still be
+ * collected. Deliberately no SIGTERM handler: installing one removes Node's
+ * default signal behaviour from the host application, and a telemetry library
+ * has no business changing how its host shuts down.
+ */
+const liveMeters = new Set();
+let exitHookInstalled = false;
+function registerForExitFlush(meter) {
+    if (typeof process === "undefined" || typeof process.on !== "function")
+        return;
+    // WeakRef is everywhere the SDK runs, but fall back rather than assume
+    const ref = typeof WeakRef === "function"
+        ? new WeakRef(meter)
+        : { deref: () => meter };
+    liveMeters.add(ref);
+    if (exitHookInstalled)
+        return;
+    exitHookInstalled = true;
+    process.on("beforeExit", () => {
+        for (const r of liveMeters) {
+            const m = r.deref();
+            if (m)
+                void m.flush();
+            else
+                liveMeters.delete(r); // collected; stop tracking it
+        }
+    });
+}
+/**
+ * Fire-and-forget token tracking. track() never throws and never blocks;
+ * events batch in memory and flush every 5s or 20 events. Network failures
+ * retry once, then drop with one console.warn — tracking can never break
+ * the host app.
+ */
+export class SpendGraph {
+    opts;
+    queue = [];
+    timer = null;
+    warned = false;
+    /** Events lost in the current run of failures, reset by a successful flush. */
+    dropped = 0;
+    /** Unpriced model ids already reported, so each is named at most once. */
+    unpricedSeen = new Set();
+    interval;
+    maxBatch;
+    /** When the pending flush was scheduled, for the suspend check in track(). */
+    timerAt = 0;
+    suspendWarned = false;
+    noKeyWarned = false;
+    streamUsageWarned = false;
+    /** The flush currently on the wire, so a later flush() can wait for it. */
+    inFlight = null;
+    constructor(opts) {
+        this.opts = opts;
+        this.interval = opts.flushIntervalMs ?? 5000;
+        this.maxBatch = opts.maxBatch ?? 20;
+        registerForExitFlush(this);
+    }
+    /**
+     * The package's client, given no retries of its own.
+     *
+     * The policy in `send` stays this class's: a flush is awaited inside
+     * serverless handlers, where blocking out a minute-long rate-limit window is
+     * worse than losing the batch, and the client's backoff would do exactly that.
+     */
+    clientFor(apiKey) {
+        return new Client({ apiKey, baseUrl: this.opts.baseUrl, attempts: 1 });
+    }
+    /** Record one LLM call. Synchronous, returns void, never throws. */
+    track(event) {
+        try {
+            if (!this.opts.apiKey) {
+                this.warnNoKey();
+                return;
+            }
+            this.detectSuspendedRuntime();
+            this.queue.push(event);
+            if (this.queue.length >= this.maxBatch) {
+                void this.flush();
+            }
+            else if (!this.timer) {
+                this.timer = setTimeout(() => void this.flush(), this.interval);
+                this.timerAt = Date.now();
+                // don't hold the process open just to flush telemetry
+                this.timer.unref?.();
+            }
+        }
+        catch {
+            /* fail-open */
+        }
+    }
+    /**
+     * Says once that tracking is off.
+     *
+     * Running without a key stays a no-op on purpose — it keeps spendgraph out of
+     * tests and local runs without branching at every call site. But an unset
+     * SPENDGRAPH_API_KEY is indistinguishable from that choice, and it is the
+     * likeliest reason a fresh integration records nothing at all: the code is
+     * wired up correctly, the dashboard is empty, and nothing anywhere says why.
+     * Every other failure in this class warns once; this was the one that stayed
+     * quiet, which made it the hardest to find.
+     */
+    warnNoKey() {
+        if (this.noKeyWarned)
+            return;
+        this.noKeyWarned = true;
+        console.warn("spendgraph: no apiKey set, so track() is recording nothing. Pass apiKey " +
+            "(usually from SPENDGRAPH_API_KEY) to start tracking, or ignore this if " +
+            "tracking is meant to be off here.");
+    }
+    /**
+     * Notices that the runtime froze with events still buffered.
+     *
+     * Serverless platforms suspend a function once it returns rather than
+     * exiting it, so neither the flush timer nor `beforeExit` ever runs and the
+     * queue is lost without a sound — the failure Langfuse documents for Lambda
+     * and Vercel. There is no reliable flag for "am I serverless", but there is
+     * direct evidence: a pending timer whose deadline passed long ago did not
+     * fire, which only happens if the runtime stopped executing between calls.
+     *
+     * Seeing that, send the stragglers now (they survive into this invocation)
+     * and say once what the fix is. Correct callers await flush(), which clears
+     * the timer, so this never fires for them.
+     */
+    detectSuspendedRuntime() {
+        if (!this.timer)
+            return;
+        // 2x the interval, so ordinary event-loop lag is never mistaken for a freeze
+        if (Date.now() - this.timerAt <= this.interval * 2)
+            return;
+        if (!this.suspendWarned) {
+            this.suspendWarned = true;
+            console.warn("spendgraph: a scheduled flush never ran — this runtime suspends between " +
+                "invocations, so buffered events are lost. Await meter.flush() before " +
+                "your handler returns.");
+        }
+        void this.flush();
+    }
+    /**
+     * Send everything buffered now. Call at the end of serverless handlers.
+     *
+     * Serialised against a flush already running, because this method is not the
+     * only thing that starts one: track() flushes by itself the moment the batch
+     * is full, and so do the timer, the suspend check and the exit hook. Those
+     * empty the queue synchronously, so an awaited flush() used to find nothing
+     * left to send and hand back an already-resolved promise while the real
+     * batch was still on the wire — and the next thing a serverless handler does
+     * after awaiting is return, freezing the runtime and losing exactly the
+     * events the await was there to save.
+     *
+     * The drain still starts synchronously — `drain()` runs as far as its first
+     * await before returning a promise, so the queue is taken during this call
+     * and not a microtask later. That ordering is load-bearing: the suspend
+     * check and the exit hook both fire a flush and then expect the buffer to be
+     * gone, and deferring it broke them.
+     */
+    async flush() {
+        const prev = this.inFlight;
+        const mine = this.drain();
+        // Both, not one after the other: they carry disjoint events, since this
+        // call already took everything the earlier one left behind.
+        const run = Promise.allSettled([prev, mine]).then(() => undefined);
+        this.inFlight = run;
+        try {
+            await run;
+        }
+        finally {
+            // Only the newest link clears the slot. An older one settling last must
+            // not strand a flush that has already queued up behind it.
+            if (this.inFlight === run)
+                this.inFlight = null;
+        }
+    }
+    /** One pass: stop the timer, take the queue, put it on the wire. */
+    async drain() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        const apiKey = this.opts.apiKey;
+        if (this.queue.length === 0 || !apiKey)
+            return;
+        const events = this.queue.splice(0, this.queue.length);
+        for (let i = 0; i < events.length; i += 100) {
+            await this.send(events.slice(i, i + 100), apiKey);
+        }
+    }
+    /**
+     * Wrap an Anthropic or OpenAI client. Use the wrapped client exactly as
+     * before — token usage is read off each response and tracked automatically.
+     *
+     * Streaming is covered too:
+     * - helper streams (`anthropic.messages.stream()`,
+     *   `openai.beta.chat.completions.stream()`) are tracked via their
+     *   final-message promise — the stream you get back is untouched;
+     * - raw streams (`create({ stream: true })`) are tee'd: you receive one
+     *   branch, usage is accumulated off the other. For OpenAI raw streams,
+     *   pass `stream_options: { include_usage: true }` or there is no usage
+     *   to read and the call goes untracked.
+     */
+    wrap(client) {
+        return this.proxy(client);
+    }
+    /**
+     * The recursive wrapper behind wrap().
+     *
+     * Two things here are deliberate and were both wrong before.
+     *
+     * `Reflect.get` is called without a receiver, so a getter runs with the real
+     * client as `this`. Passing the proxy — the obvious reading of the Proxy
+     * docs — makes any getter that touches a `#private` field throw
+     * "Cannot read private member", and the provider SDKs this is built to wrap
+     * use private fields. That turned a tracking wrapper into a crash on a
+     * client that works perfectly well unwrapped, in an SDK whose whole contract
+     * is to fail open.
+     *
+     * And each wrapper is kept, so `client.messages` and `client.messages.create`
+     * are the same object and the same function every time they are read.
+     * Rebuilding them per access allocated a proxy and a closure on every call
+     * and quietly broke any caller that holds onto a method or compares
+     * identity. The cache is per proxy and keyed by property, so a method always
+     * applies to the object it was read from, and it re-wraps if the underlying
+     * value is ever replaced.
+     */
+    proxy(target) {
+        const cache = new Map();
+        return new Proxy(target, {
+            get: (obj, prop) => {
+                const value = Reflect.get(obj, prop);
+                if (value === null || (typeof value !== "function" && typeof value !== "object")) {
+                    return value;
+                }
+                const hit = cache.get(prop);
+                if (hit && hit.from === value)
+                    return hit.out;
+                const out = typeof value === "function"
+                    ? (...args) => this.observeResult(value.apply(obj, args), args)
+                    : this.proxy(value);
+                cache.set(prop, { from: value, out });
+                return out;
+            },
+        });
+    }
+    observeResult(result, args) {
+        try {
+            // Helper streams return synchronously and expose a final-message
+            // promise (Anthropic MessageStream / OpenAI ChatCompletionStream).
+            // Awaiting it does not consume the caller's iterator.
+            const helper = result;
+            const final = typeof helper?.finalMessage === "function"
+                ? helper.finalMessage()
+                : typeof helper?.finalChatCompletion === "function"
+                    ? helper.finalChatCompletion()
+                    : null;
+            if (final instanceof Promise) {
+                final.then((m) => this.trackFromResponse(m), () => { });
+                return result;
+            }
+            if (result instanceof Promise) {
+                // Raw streaming (create({stream: true})) resolves to an SSE stream;
+                // swap in a tee'd branch so we can read usage without consuming the
+                // caller's. Only then do we replace the promise — non-streaming
+                // calls keep the SDK's original promise (withResponse() etc.).
+                const wantsStream = !!args[0]?.stream;
+                if (wantsStream) {
+                    return result.then((v) => this.interceptStream(v));
+                }
+                // provider errors are the caller's to handle
+                result.then((v) => this.trackFromResponse(v), () => { });
+            }
+        }
+        catch {
+            /* fail-open */
+        }
+        return result;
+    }
+    interceptStream(v) {
+        try {
+            const s = v;
+            if (s && typeof s.tee === "function" && typeof s[Symbol.asyncIterator] === "function") {
+                const [mine, theirs] = s.tee();
+                void this.consumeStream(mine);
+                return theirs;
+            }
+            this.trackFromResponse(v);
+        }
+        catch {
+            /* fail-open */
+        }
+        return v;
+    }
+    /** Accumulate usage off a tee'd SSE branch (Anthropic events / OpenAI chunks). */
+    async consumeStream(iter) {
+        try {
+            let model;
+            let inputTokens;
+            let outputTokens;
+            let cacheReadTokens = 0;
+            let cacheWriteTokens = 0;
+            let citationTokens;
+            let reasoningTokens;
+            let sawChunk = false;
+            for await (const raw of iter) {
+                sawChunk = true;
+                const ev = raw;
+                if (ev?.type === "message_start" && ev.message) {
+                    // Anthropic: input + cache usage arrive up front
+                    model = ev.message.model ?? model;
+                    const u = ev.message.usage ?? {};
+                    inputTokens = u.input_tokens ?? inputTokens;
+                    cacheReadTokens = u.cache_read_input_tokens ?? 0;
+                    cacheWriteTokens = u.cache_creation_input_tokens ?? 0;
+                }
+                else if (ev?.type === "message_delta" && ev.usage) {
+                    // Anthropic: cumulative output count
+                    outputTokens = ev.usage.output_tokens ?? outputTokens;
+                }
+                else if (ev?.object === "chat.completion.chunk") {
+                    // OpenAI: usage only on the final chunk, and only with
+                    // stream_options: { include_usage: true }
+                    model = ev.model ?? model;
+                    if (ev.usage) {
+                        const u = ev.usage;
+                        const cached = u.prompt_tokens_details?.cached_tokens ?? 0;
+                        inputTokens = Math.max(0, (u.prompt_tokens ?? 0) - cached);
+                        cacheReadTokens = cached;
+                        outputTokens = u.completion_tokens;
+                        citationTokens = u.citation_tokens;
+                        reasoningTokens = u.reasoning_tokens;
+                    }
+                }
+            }
+            if (model && typeof inputTokens === "number" && typeof outputTokens === "number") {
+                this.track({
+                    model,
+                    inputTokens,
+                    outputTokens,
+                    cacheReadTokens,
+                    cacheWriteTokens,
+                    citationTokens,
+                    reasoningTokens,
+                });
+            }
+            else if (sawChunk) {
+                this.warnStreamNoUsage();
+            }
+        }
+        catch {
+            /* fail-open — never disturb the caller's branch */
+        }
+    }
+    /**
+     * Says once that a streamed call went untracked.
+     *
+     * A raw OpenAI stream carries no usage unless the caller asks for it, so the
+     * accumulator above finishes with nothing to report and simply returns —
+     * correct, and completely silent. That is the same hole warnNoKey was added
+     * to close: the integration looks right, the dashboard stays empty for those
+     * calls, and nothing anywhere connects the two. Anthropic streams always
+     * carry usage, so in practice this names the one option that fixes it.
+     *
+     * Guarded on having seen at least one chunk, so an empty or aborted stream —
+     * where there was never anything to read — stays quiet.
+     */
+    warnStreamNoUsage() {
+        if (this.streamUsageWarned)
+            return;
+        this.streamUsageWarned = true;
+        console.warn("spendgraph: a streamed response carried no token usage, so that call " +
+            "was not tracked. For OpenAI raw streams pass " +
+            "stream_options: { include_usage: true }. Further untracked streams " +
+            "are not logged.");
+    }
+    trackFromResponse(res) {
+        try {
+            if (!res || typeof res !== "object")
+                return;
+            const r = res;
+            if (!r.usage || !r.model)
+                return;
+            const u = r.usage;
+            if (typeof u.input_tokens === "number" && typeof u.output_tokens === "number") {
+                // Anthropic: cache tokens are separate fields already
+                this.track({
+                    model: r.model,
+                    inputTokens: u.input_tokens,
+                    outputTokens: u.output_tokens,
+                    cacheReadTokens: u.cache_read_input_tokens ?? 0,
+                    cacheWriteTokens: u.cache_creation_input_tokens ?? 0,
+                });
+                return;
+            }
+            if (typeof u.prompt_tokens === "number" && typeof u.completion_tokens === "number") {
+                // OpenAI: cached tokens are included in prompt_tokens — split them out
+                const cached = u.prompt_tokens_details?.cached_tokens ?? 0;
+                this.track({
+                    model: r.model,
+                    inputTokens: Math.max(0, u.prompt_tokens - cached),
+                    outputTokens: u.completion_tokens,
+                    cacheReadTokens: cached,
+                    citationTokens: u.citation_tokens,
+                    reasoningTokens: u.reasoning_tokens,
+                });
+            }
+        }
+        catch {
+            /* fail-open */
+        }
+    }
+    async send(events, apiKey, attempt = 0) {
+        let body;
+        try {
+            body = await this.clientFor(apiKey).post("/api/v1/ingest", { events });
+        }
+        catch (err) {
+            return this.onSendFailed(err, events, apiKey, attempt);
+        }
+        // A run of failures is over; a later one is news again rather than more
+        // of the same, so the warning is allowed to fire once more.
+        this.warned = false;
+        this.dropped = 0;
+        this.reportUnpriced(body);
+    }
+    /**
+     * The drop policy, which is this class's rather than the client's.
+     *
+     * 429 is the one failure the server tells you how to fix: it sends
+     * retry-after, and the window is at most a minute wide, so a batch that
+     * arrived at the end of one is deliverable a second later. Dropping it
+     * immediately threw that away and reported the least useful thing it could —
+     * "ingest returned 429" — for the one status with a documented remedy. The
+     * wait is capped because flush() is awaited inside serverless handlers, where
+     * blocking for a minute is worse than losing the batch.
+     */
+    async onSendFailed(err, events, apiKey, attempt) {
+        const failure = err instanceof SpendgraphError ? err : null;
+        if (!failure || failure.status === 0) {
+            if (attempt === 0)
+                return this.send(events, apiKey, 1);
+            this.reportDropped(events.length, `unreachable (${String(err)})`);
+            return;
+        }
+        if (failure.status >= 500 && attempt === 0) {
+            return this.send(events, apiKey, 1);
+        }
+        if (failure.status === 429 && attempt === 0) {
+            const waitMs = failure.retryAfterMs;
+            if (waitMs !== null && waitMs <= MAX_RETRY_WAIT_MS) {
+                await new Promise((r) => setTimeout(r, waitMs));
+                return this.send(events, apiKey, 1);
+            }
+            this.reportDropped(events.length, waitMs === null
+                ? "rate limited by ingest"
+                : `rate limited by ingest, clear in ${Math.ceil(waitMs / 1000)}s`);
+            return;
+        }
+        this.reportDropped(events.length, `ingest returned ${failure.status}`);
+    }
+    /**
+     * Surfaces model ids the server could not price, once each.
+     *
+     * These are accepted and stored, so nothing here is an error — but they cost
+     * $0, and a dashboard reading $0 is indistinguishable from one reading
+     * "nothing happened". Naming the id in the integrator's own console is the
+     * cheapest possible moment to catch a typo or an unmapped model, and the
+     * per-id guard keeps a steady stream of the same unknown model from becoming
+     * log noise.
+     */
+    reportUnpriced(body) {
+        {
+            const parsed = body;
+            if (!Array.isArray(parsed?.unpricedModels))
+                return;
+            const fresh = parsed.unpricedModels
+                .filter((m) => typeof m === "string")
+                .filter((m) => !this.unpricedSeen.has(m));
+            if (fresh.length === 0)
+                return;
+            for (const m of fresh)
+                this.unpricedSeen.add(m);
+            console.warn(`spendgraph: no price for ${fresh.map((m) => `"${m}"`).join(", ")}. ` +
+                `These events are recorded but cost $0 until the model is in the catalog.`);
+        }
+    }
+    /**
+     * Warns once per outage, with a running count of what was lost.
+     *
+     * Warning on every flush would spam a hot loop, but warning exactly once per
+     * process — the previous behaviour — hid a server-side bug that failed every
+     * full batch: one line early in a long-lived process, then silence, while the
+     * dashboard quietly undercounted. The count is what makes the silence legible
+     * when someone does go looking.
+     */
+    reportDropped(count, reason) {
+        this.dropped += count;
+        if (this.warned)
+            return;
+        this.warned = true;
+        console.warn(`spendgraph: ${reason}; dropped ${this.dropped} event(s). ` +
+            `Further drops are counted but not logged until a flush succeeds.`);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spendgraph/sdk",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Track LLM input/output tokens and cost. Three functions, zero dependencies, fail-open.",
   "license": "MIT",
   "repository": {
@@ -43,7 +43,8 @@
     "README.md"
   ],
   "scripts": {
-    "build": "tsc -p tsconfig.json"
+    "build": "tsc -p tsconfig.json",
+    "test": "vitest run"
   },
   "devDependencies": {
     "typescript": "^5"