anpord 0.1.2 → 0.1.4

package/dist/evals-api-sqVNeWet.d.cts

@@ -0,0 +1,93 @@
+import { Schema } from "effect";
+import "@effect/platform";
+//#region ../schema/src/domain/errors.d.ts
+declare const NotFound_base: Schema.TaggedErrorClass<NotFound, "NotFound", {
+  readonly _tag: Schema.tag<"NotFound">;
+} & {
+  message: typeof Schema.String;
+}>;
+declare class NotFound extends NotFound_base {}
+declare const Conflict_base: Schema.TaggedErrorClass<Conflict, "Conflict", {
+  readonly _tag: Schema.tag<"Conflict">;
+} & {
+  message: typeof Schema.String;
+}>;
+declare class Conflict extends Conflict_base {}
+declare const BadRequest_base: Schema.TaggedErrorClass<BadRequest, "BadRequest", {
+  readonly _tag: Schema.tag<"BadRequest">;
+} & {
+  message: typeof Schema.String;
+}>;
+declare class BadRequest extends BadRequest_base {}
+declare const Unauthorized_base: Schema.TaggedErrorClass<Unauthorized, "Unauthorized", {
+  readonly _tag: Schema.tag<"Unauthorized">;
+} & {
+  message: typeof Schema.String;
+}>;
+declare class Unauthorized extends Unauthorized_base {}
+declare const Forbidden_base: Schema.TaggedErrorClass<Forbidden, "Forbidden", {
+  readonly _tag: Schema.tag<"Forbidden">;
+} & {
+  message: typeof Schema.String;
+}>;
+/** Distinct from Unauthorized: the caller is who they say they are, and the
+ * answer is still no. Signing in again cannot help, so the client must not
+ * treat this as an expired session. */
+declare class Forbidden extends Forbidden_base {}
+//#endregion
+//#region ../schema/src/public/evals-api.d.ts
+declare const PublicStartEvalRequest: Schema.Struct<{
+  cases: Schema.filter<Schema.Array$<Schema.refine<{
+    readonly name: string;
+    readonly setup?: string | null | undefined;
+    readonly source?: {
+      readonly kind: "empty";
+    } | {
+      readonly kind: "repo";
+      readonly ref: string | null;
+      readonly url: string;
+    } | {
+      readonly kind: "files";
+      readonly files: {
+        readonly [x: string]: string;
+      };
+    } | undefined;
+    readonly validator?: {
+      readonly name: string;
+      readonly source: string;
+    } | null | undefined;
+    readonly variables?: {
+      readonly [x: string]: string;
+    } | undefined;
+    readonly verify: string | null;
+  }, Schema.Struct<{
+    name: typeof Schema.String;
+    setup: Schema.optional<Schema.NullOr<typeof Schema.String>>;
+    source: Schema.optional<Schema.Union<[Schema.Struct<{
+      kind: Schema.Literal<["empty"]>;
+    }>, Schema.Struct<{
+      kind: Schema.Literal<["repo"]>;
+      ref: Schema.NullOr<typeof Schema.String>;
+      url: Schema.refine<string, typeof Schema.String>;
+    }>, Schema.Struct<{
+      files: Schema.Record$<typeof Schema.String, typeof Schema.String>;
+      kind: Schema.Literal<["files"]>;
+    }>]>>;
+    validator: Schema.optional<Schema.NullOr<Schema.Struct<{
+      name: Schema.filter<Schema.filter<typeof Schema.String>>;
+      source: Schema.filter<Schema.filter<typeof Schema.String>>;
+    }>>>;
+    variables: Schema.optional<Schema.Record$<typeof Schema.String, typeof Schema.String>>;
+    verify: Schema.NullOr<typeof Schema.String>;
+  }>>>>;
+  prompt: typeof Schema.String;
+  tasks: Schema.filter<Schema.Array$<Schema.Struct<{
+    harness: Schema.Literal<["codex", "opencode", "pi", "fx", "claude", "gemini", "qwen", "cursor"]>;
+    model: Schema.filter<typeof Schema.String>;
+    provider: Schema.Literal<["daytona", "e2b", "upstash", "modal", "cloudflare", "vercel"]>;
+  }>>>;
+  trials: Schema.filter<typeof Schema.Int>;
+}>;
+type PublicStartEvalRequest = typeof PublicStartEvalRequest.Type;
+//#endregion
+export { NotFound as a, Forbidden as i, BadRequest as n, Unauthorized as o, Conflict as r, PublicStartEvalRequest as t };

package/dist/index.cjs

@@ -1,8 +1,373 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-Cno4LwRq.cjs");
-const require_errors = require("./errors-C0E5A8fN.cjs");
+const require_client = require("./client-BCZEcfVE.cjs");
+const require_errors = require("./errors-Boum34f5.cjs");
+const require_source = require("./source-DvlGawEw.cjs");
 let _effect_platform = require("@effect/platform");
 let effect = require("effect");
+//#region ../template/src/errors.ts
+/**
+* A template asked for something the caller did not supply.
+*
+* A plain `Error` rather than a tagged one: this package has no dependencies,
+* and it is thrown across the published SDK boundary where callers reach for
+* `instanceof`.
+*/
+var MissingVariables = class extends Error {
+	missing;
+	constructor(missing) {
+		const names = missing.map((name) => `{{${name}}}`).join(", ");
+		super(`No value for ${names}. Pass them, or set onMissing to "keep" or "empty".`);
+		this.name = "MissingVariables";
+		this.missing = missing;
+	}
+};
+//#endregion
+//#region ../template/src/render.ts
+/** Null and undefined read as absent rather than as the strings "null" and
+* "undefined", because that is what `{ name: user.name }` produces when the
+* field is not there. */
+const suppliedValue = (values, name) => {
+	const value = values[name];
+	return value === null || value === void 0 ? void 0 : String(value);
+};
+/**
+* Fills `{{name}}` from a flat map.
+*
+* Dots and hyphens are part of the name rather than a path: `{{user.name}}`
+* reads the key `"user.name"`. One rule beats two, and a caller who wants
+* nesting writes the flat key themselves.
+*/
+function render(template, values, options = {}) {
+	const onMissing = options.onMissing ?? "throw";
+	const missing = [];
+	const content = template.replace(require_errors.tokenMatcher(), (match, open, close, name) => {
+		if (open !== void 0) return "{{";
+		if (close !== void 0) return "}}";
+		if (name === void 0) return match;
+		const value = suppliedValue(values, name);
+		if (value !== void 0) return value;
+		if (!missing.includes(name)) missing.push(name);
+		return onMissing === "empty" ? "" : match;
+	});
+	if (onMissing === "throw" && missing.length > 0) throw new MissingVariables(missing);
+	return {
+		content,
+		missing
+	};
+}
+//#endregion
+//#region src/client/cache/cache-key.ts
+/** Structural rather than referential, so two identical selectors are one
+* entry and one in-flight request rather than two. A plain object literal
+* hashes by reference and every call would miss. */
+const cacheKey = (selector) => effect.Data.struct({
+	channel: selector.channel,
+	id: selector.id,
+	includeVersions: selector.includeVersions,
+	version: selector.version
+});
+//#endregion
+//#region src/client/cache/keys.ts
+const NAMESPACE = "prompt";
+/**
+* Which version a selector asks for.
+*
+* Asking for no channel is its own selector rather than a named one: the
+* organisation decides which channel answers, it can rename or drop that
+* channel, and naming a guess here would key the answer under a channel the
+* server never resolved.
+*/
+const selectorSegment = (selector) => {
+	if (selector.version !== void 0) return `v:${selector.version}`;
+	return selector.channel === void 0 ? "default" : `c:${selector.channel}`;
+};
+/**
+* What makes one cached answer different from another.
+*
+* `includeVersions` belongs here even though the server's key omits it: this
+* cache holds the response rather than the resolution, and a response asked
+* for without history has no `versions` to give to a caller who wanted it.
+*/
+const promptKey = (selector) => [
+	NAMESPACE,
+	encodeURIComponent(selector.id),
+	selectorSegment(selector),
+	selector.includeVersions === true ? "full" : "bare"
+].join(":");
+/** Every key for one prompt, so a write invalidates all of them. */
+const promptPrefix = (id) => `${NAMESPACE}:${encodeURIComponent(id)}:`;
+//#endregion
+//#region src/client/cache/prompt-cache.ts
+var PromptCache = class extends effect.Context.Tag("@anpord/sdk/PromptCache")() {};
+const make$1 = (settings, fetch) => effect.Effect.gen(function* () {
+	const scope = yield* effect.Effect.scope;
+	const permits = yield* effect.Effect.makeSemaphore(settings.maxConcurrentRefresh);
+	const cache = yield* effect.Cache.makeWith({
+		capacity: settings.capacity,
+		lookup: (selector) => fetch(selector),
+		/** A failure is not an answer, so it is not kept: the next call tries
+		* again rather than waiting out a whole window on one bad response. */
+		timeToLive: (exit) => effect.Exit.isFailure(exit) ? effect.Duration.zero : effect.Duration.infinity
+	});
+	/** A pinned version cannot change, so only capacity ever evicts it. A
+	* channel can be repointed at any moment. */
+	const ttlFor = (selector) => selector.version === void 0 ? settings.ttlMs : Number.POSITIVE_INFINITY;
+	const ageOf = (key) => effect.Effect.gen(function* () {
+		const stats = yield* cache.entryStats(key);
+		if (effect.Option.isNone(stats)) return effect.Option.none();
+		const now = yield* effect.Clock.currentTimeMillis;
+		return effect.Option.some(Math.max(0, now - stats.value.loadedMillis));
+	});
+	/** Forked into the cache's own scope rather than as a daemon, so a client
+	* that is disposed takes its refreshes with it. A refresh that cannot get
+	* a permit is dropped: the next call will ask again. */
+	const revalidate = (key) => permits.withPermitsIfAvailable(1)(cache.refresh(key)).pipe(effect.Effect.catchAllCause(() => effect.Effect.void), effect.Effect.forkIn(scope), effect.Effect.asVoid);
+	const heldWithin = (selector, limitMs) => effect.Effect.gen(function* () {
+		const key = cacheKey(selector);
+		const ttl = ttlFor(key);
+		const value = yield* cache.getOptionComplete(key);
+		const age = yield* ageOf(key);
+		if (effect.Option.isNone(value) || effect.Option.isNone(age)) return effect.Option.none();
+		if (age.value > limitMs) return effect.Option.none();
+		if (age.value > ttl) yield* revalidate(key);
+		return effect.Option.some({
+			ageMs: age.value,
+			value: value.value
+		});
+	});
+	return {
+		held: (selector) => heldWithin(selector, settings.maxStaleMs),
+		invalidate: (id) => effect.Effect.gen(function* () {
+			const prefix = promptPrefix(id);
+			const keys = yield* cache.keys;
+			yield* effect.Effect.forEach(keys.filter((key) => promptKey(key).startsWith(prefix)), (key) => cache.invalidate(key), { discard: true });
+		}).pipe(effect.Effect.withSpan("PromptCache.invalidate")),
+		load: (selector) => effect.Effect.gen(function* () {
+			const key = cacheKey(selector);
+			const value = yield* fetch(selector);
+			yield* cache.set(key, value);
+			return value;
+		}).pipe(effect.Effect.withSpan("PromptCache.load", { attributes: { key: promptKey(selector) } })),
+		stale: (selector) => heldWithin(selector, Number.POSITIVE_INFINITY)
+	};
+});
+const layer$1 = (settings, fetch) => effect.Layer.scoped(PromptCache, make$1(settings, fetch));
+//#endregion
+//#region src/client/cache/noop.ts
+/** Disabled caching is a cache that holds nothing, so the one path through
+* `resolvePrompt` still runs rather than being wrapped in a condition. */
+const noopLayer = (fetch) => effect.Layer.succeed(PromptCache, {
+	/** Effects rather than bare values: yielding an Option would make an empty
+	* one a failure, and every read would end in NoSuchElementException. */
+	held: () => effect.Effect.succeedNone,
+	invalidate: () => effect.Effect.void,
+	load: fetch,
+	stale: () => effect.Effect.succeedNone
+});
+//#endregion
+//#region src/client/cache/failure.ts
+/** A status the server chose deliberately, telling the caller something true
+* about their request rather than about its own health. */
+const DEFINITIVE = /* @__PURE__ */ new Set([
+	400,
+	401,
+	403,
+	404,
+	409,
+	422
+]);
+/**
+* Whether a failure means the answer is unavailable rather than no.
+*
+* A 404 is a successful reply carrying real information: this prompt does not
+* exist. Covering it with a fallback turns a typo in a prompt id into a silent
+* permanent substitution that ships and is found weeks later. Serving stale
+* content covers the case where a prompt someone was using is removed, and it
+* sits ahead of the fallback anyway.
+*
+* An unrecognised failure counts as unavailable: a transport error carries no
+* status at all, and guessing the other way would turn a flaky network into an
+* outage.
+*/
+const isAvailabilityFailure = (error) => error.status === void 0 || !DEFINITIVE.has(error.status);
+//#endregion
+//#region src/client/cache/fallback.ts
+/** No version the API can return is zero, so a caller who logs it knows
+* without being told that they are reading a fallback rather than a prompt. */
+const FALLBACK_VERSION = 0;
+/**
+* The shape of a prompt, built from what the caller had to hand.
+*
+* The branded fields are asserted rather than decoded: this value never came
+* from the API and never goes back to it, and a fallback that could fail to
+* parse would defeat the reason it exists.
+*/
+const fallbackPrompt = (selector, fallback, now) => {
+	const given = typeof fallback === "string" ? { content: fallback } : fallback;
+	return {
+		channel: selector.channel ?? null,
+		config: given.config ?? {},
+		content: given.content,
+		createdAt: effect.DateTime.unsafeMake(now),
+		id: selector.id,
+		message: null,
+		name: selector.id,
+		version: FALLBACK_VERSION
+	};
+};
+//#endregion
+//#region src/client/cache/resolve.ts
+/**
+* Which answer the caller gets, in the order the answers are worth having.
+*
+* Fresh cache first, because that is the whole point. Then the API, since it
+* is the only source that can be right. Then whatever is held however old,
+* because a prompt someone wrote and promoted beats a string written at deploy
+* time. Then the fallback, which the caller asked for. Then the failure, since
+* with nothing cached and no fallback there is no honest answer to give.
+*/
+const resolvePrompt = (options) => effect.Effect.gen(function* () {
+	const cache = yield* PromptCache;
+	const key = promptKey(options);
+	const held = yield* cache.held(options);
+	if (effect.Option.isSome(held)) return {
+		metadata: {
+			ageMs: held.value.ageMs,
+			/** Where the answer came from, not how old it is. An entry read in
+			* the same millisecond it was stored is still an entry, and calling
+			* it fresh made a cache hit indistinguishable from a network read. */
+			freshness: "cached",
+			key
+		},
+		value: held.value.value
+	};
+	const loaded = yield* effect.Effect.either(cache.load(options));
+	if (loaded._tag === "Right") return {
+		metadata: {
+			ageMs: 0,
+			freshness: "fresh",
+			key
+		},
+		value: loaded.right
+	};
+	const error = require_errors.asAnpordError(loaded.left);
+	if (!isAvailabilityFailure(error)) return yield* effect.Effect.fail(error);
+	const remembered = yield* cache.stale(options);
+	if (effect.Option.isSome(remembered)) return {
+		metadata: {
+			ageMs: remembered.value.ageMs,
+			freshness: "stale",
+			key,
+			reason: error.message
+		},
+		value: remembered.value.value
+	};
+	if (options.fallback === void 0) return yield* effect.Effect.fail(error);
+	const now = yield* effect.Clock.currentTimeMillis;
+	return {
+		metadata: {
+			ageMs: 0,
+			freshness: "fallback",
+			key,
+			reason: error.message
+		},
+		value: fallbackPrompt(options, options.fallback, now)
+	};
+}).pipe(effect.Effect.withSpan("Prompts.resolve", { attributes: { key: promptKey(options) } }));
+//#endregion
+//#region src/client/cache/settings.ts
+/**
+* How long a channel's answer is trusted without asking again.
+*
+* This is the delay between a promotion and callers seeing it, so it is short:
+* the dashboard tells whoever promotes that callers receive the new version
+* immediately, and a minute of drift makes that untrue exactly when someone is
+* rolling back something bad. Serving stale while refreshing means the number
+* costs no latency, only freshness.
+*/
+const TTL_MS = 15e3;
+/** Long enough to cover a deploy or an incident, short enough that a value
+* this old is worth saying out loud. Only reached while the API answers; when
+* it cannot be reached at all there is no bound, since a day-old real prompt
+* beats no prompt. */
+const MAX_STALE_MS = 864e5;
+/** Least recently used past this. A thousand prompts is far more than one
+* process serves and still bounds the memory a long-running server holds. */
+const CAPACITY = 1e3;
+/** A cold cache going stale at once must not fork a refresh per prompt. */
+const MAX_CONCURRENT_REFRESH = 8;
+const positive = (value, fallback) => value !== void 0 && Number.isFinite(value) && value > 0 ? value : fallback;
+const settingsFrom = (options) => {
+	const given = typeof options === "object" ? options : {};
+	return {
+		capacity: positive(given.capacity, CAPACITY),
+		maxConcurrentRefresh: positive(given.maxConcurrentRefresh, MAX_CONCURRENT_REFRESH),
+		maxStaleMs: positive(given.maxStaleMs, MAX_STALE_MS),
+		ttlMs: positive(given.ttlMs, TTL_MS)
+	};
+};
+const cacheEnabled = (options) => options !== false;
+//#endregion
+//#region src/client/promised.ts
+const promised = (group) => Object.fromEntries(Object.entries(group).map(([method, run]) => [method, async (request) => {
+	const exit = await effect.Effect.runPromiseExit(run({ payload: request ?? {} }));
+	if (effect.Exit.isSuccess(exit)) return exit.value;
+	throw require_errors.asAnpordError(effect.Cause.failureOption(exit.cause).pipe(effect.Option.getOrElse(() => effect.Cause.squash(exit.cause))));
+}]));
+//#endregion
+//#region src/client/wait.ts
+var EvalTimeout = class extends Error {
+	name = "EvalTimeout";
+	runId;
+	constructor(runId, elapsedMs) {
+		const elapsed = elapsedMs < 1e3 ? `${elapsedMs}ms` : `${Math.round(elapsedMs / 1e3)}s`;
+		super(`Eval ${runId} was still running after ${elapsed}. It was not cancelled — read it later with evals.get({ id: "${runId}" }).`);
+		this.runId = runId;
+	}
+};
+var EvalAborted = class extends Error {
+	name = "EvalAborted";
+	runId;
+	constructor(runId) {
+		super(`Waiting on eval ${runId} was aborted. The run was not cancelled — read it later with evals.get({ id: "${runId}" }).`);
+		this.runId = runId;
+	}
+};
+const DEFAULT_INTERVAL = 2e3;
+const DEFAULT_MAX_INTERVAL = 1e4;
+const DEFAULT_TIMEOUT = 36e5;
+const nextInterval = (current, max) => Math.min(Math.round(current * 1.5), max);
+const sleep = (ms, signal) => new Promise((resolve, reject) => {
+	const timer = setTimeout(() => {
+		signal?.removeEventListener("abort", onAbort);
+		resolve();
+	}, ms);
+	function onAbort() {
+		clearTimeout(timer);
+		reject(signal?.reason);
+	}
+	signal?.addEventListener("abort", onAbort, { once: true });
+});
+const waitForRun = async (get, id, options = {}) => {
+	const { maxIntervalMs = DEFAULT_MAX_INTERVAL, onProgress, pollIntervalMs = DEFAULT_INTERVAL, signal, timeoutMs = DEFAULT_TIMEOUT } = options;
+	const startedAt = Date.now();
+	let interval = pollIntervalMs;
+	for (;;) {
+		if (signal?.aborted) throw new EvalAborted(id);
+		const run = await get({ id });
+		onProgress?.(run);
+		if (run.status !== "running") return run;
+		const elapsed = Date.now() - startedAt;
+		if (elapsed >= timeoutMs) throw new EvalTimeout(id, elapsed);
+		try {
+			await sleep(Math.min(interval, timeoutMs - elapsed), signal);
+		} catch {
+			throw new EvalAborted(id);
+		}
+		interval = nextInterval(interval, maxIntervalMs);
+	}
+};
+//#endregion
 //#region src/client/anpord.ts
 const resolveApiKey = (provided) => {
 	const apiKey = provided ?? globalThis.process?.env?.ANPORD_API_KEY;
@@ -10,26 +375,81 @@ const resolveApiKey = (provided) => {
 	return apiKey;
 };
 var Anpord = class {
+	evals;
 	prompts;
+	runtime;
 	constructor(options = {}) {
 		const apiKey = resolveApiKey(options.apiKey);
 		const client = require_client.make({
 			apiKey: effect.Redacted.make(apiKey),
 			baseUrl: options.baseUrl ?? "https://api.anpord.com"
 		}).pipe(effect.Effect.provide(_effect_platform.FetchHttpClient.layer), effect.Effect.runSync);
-		this.prompts = promised(client.prompts);
+		const fetch = (selector) => client.prompts.get({ payload: selector });
+		this.runtime = effect.ManagedRuntime.make(cacheEnabled(options.cache) ? layer$1(settingsFrom(options.cache), fetch) : noopLayer(fetch));
+		const group = promised(client.prompts);
+		const evals = promised(client.evals);
+		this.evals = {
+			...evals,
+			startAndWait: async (options) => {
+				const { maxIntervalMs, onProgress, pollIntervalMs, signal, timeoutMs, ...request } = options;
+				const { id } = await evals.start(request);
+				return await waitForRun(evals.get, id, {
+					maxIntervalMs,
+					onProgress,
+					pollIntervalMs,
+					signal,
+					timeoutMs
+				});
+			},
+			wait: ({ id, ...options }) => waitForRun(evals.get, id, options)
+		};
+		const forget = (id) => this.runtime.runPromise(effect.Effect.flatMap(PromptCache, (cache) => cache.invalidate(id))).catch(() => void 0);
+		this.prompts = {
+			...group,
+			get: (request) => this.resolve(request),
+			promote: invalidating(group.promote, forget),
+			update: invalidating(group.update, forget)
+		};
+	}
+	dispose() {
+		return this.runtime.dispose();
+	}
+	[Symbol.asyncDispose]() {
+		return this.runtime.dispose();
+	}
+	async resolve(options) {
+		const exit = await this.runtime.runPromiseExit(resolvePrompt(options));
+		if (effect.Exit.isFailure(exit)) throw require_errors.asAnpordError(effect.Cause.failureOption(exit.cause).pipe(effect.Option.getOrElse(() => effect.Cause.squash(exit.cause))));
+		const prompt = exit.value.value;
+		const value = options.variables === void 0 ? { ...prompt } : {
+			...prompt,
+			content: render(prompt.content, options.variables).content
+		};
+		return Object.defineProperty(value, "anpord", {
+			enumerable: false,
+			value: exit.value.metadata
+		});
 	}
 };
-const promised = (group) => Object.fromEntries(Object.entries(group).map(([method, run]) => [method, async (request) => {
-	const exit = await effect.Effect.runPromiseExit(run({ payload: request ?? {} }));
-	if (effect.Exit.isSuccess(exit)) return exit.value;
-	throw require_errors.asAnpordError(effect.Cause.failureOption(exit.cause).pipe(effect.Option.getOrElse(() => effect.Cause.squash(exit.cause))));
-}]));
+const invalidating = (write, forget) => async (request) => {
+	const result = await write(request);
+	await forget(request.id);
+	return result;
+};
+//#endregion
+//#region src/evals/define.ts
+const defineEval = (definition) => definition;
 //#endregion
 exports.Anpord = Anpord;
 exports.AnpordApi = require_client.AnpordApi;
 exports.AnpordError = require_errors.AnpordError;
 exports.DEFAULT_BASE_URL = require_client.DEFAULT_BASE_URL;
+exports.EvalAborted = EvalAborted;
+exports.EvalTimeout = EvalTimeout;
 exports.MissingApiKey = require_errors.MissingApiKey;
+exports.defineEval = defineEval;
+exports.empty = require_source.empty;
+exports.files = require_source.files;
 exports.layer = require_client.layer;
 exports.make = require_client.make;
+exports.repo = require_source.repo;