@posthog/ai 8.15.2 → 8.16.0

package/dist/evaluations/index.d.ts

@@ -0,0 +1,497 @@
+//#region src/evaluations/serialization.d.ts
+/** JSON accepted in evaluation payloads. No implicit toJSON or string conversion. */
+type JSONValue = null | boolean | number | string | JSONValue[] | {
+  [key: string]: JSONValue;
+};
+type JSONObject = {
+  [key: string]: JSONValue;
+};
+/** Large signed-64-bit counts use bigint; safe counts are returned as numbers. */
+type EvaluationCount = number | bigint;
+type WireObject = Record<string, unknown>;
+interface Page<T> {
+  count: EvaluationCount;
+  results: T[];
+  nextOffset: number | null;
+}
+//#endregion
+//#region src/evaluations/transport.d.ts
+interface RequestOptions {
+  signal?: AbortSignal;
+}
+interface OfflineEvaluationsOptions {
+  projectId: number;
+  secretKey: string;
+  host?: string;
+  /** Per-attempt timeout, including reading the response body. Default: 15,000 ms. */
+  timeoutMs?: number;
+  /** Additional attempts for operations with safe replay identity. Default: 3. */
+  maxRetries?: number;
+}
+declare class Transport {
+  #private;
+  readonly projectId: number;
+  constructor(options: OfflineEvaluationsOptions);
+  close(): void;
+  request(method: string, path: string, request?: RequestOptions & {
+    body?: string;
+    params?: WireObject;
+    retrySafe?: boolean;
+  }): Promise<WireObject>;
+}
+//#endregion
+//#region src/evaluations/datasets.d.ts
+interface Audit {
+  createdAt: string;
+  createdBy: JSONObject | null;
+  teamId: number;
+}
+interface Dataset extends Audit {
+  id: string;
+  name: string;
+  description: string;
+  metadata: JSONObject;
+  tags: string[];
+  archived: boolean;
+  currentRevision: number | null;
+  currentRevisionId: string | null;
+  updatedAt: string | null;
+}
+interface DatasetRevision extends Audit {
+  id: string;
+  datasetId: string;
+  revision: number;
+}
+interface DatasetItem extends Audit {
+  id: string;
+  datasetId: string;
+  clientItemId: string | null;
+  version: number;
+  versionId: string;
+  datasetRevision: number;
+  datasetRevisionId: string;
+  archived: boolean;
+  input: JSONValue;
+  expectedOutput: JSONValue;
+  sourceOutput: JSONValue;
+  metadata: JSONObject;
+  sourceTraceId: string | null;
+  sourceEventId: string | null;
+  sourceTimestamp: string | null;
+  updatedAt: string | null;
+  versionCreatedAt: string;
+  versionCreatedBy: JSONObject | null;
+}
+type DatasetPage = Page<Dataset>;
+type DatasetRevisionPage = Page<DatasetRevision>;
+type DatasetItemPage = Page<DatasetItem>;
+type DatasetOrder = 'created_at' | '-created_at' | 'updated_at' | '-updated_at';
+interface DatasetMetadata {
+  name?: string;
+  description?: string;
+  metadata?: JSONObject;
+  tags?: readonly string[];
+}
+interface PageOptions extends RequestOptions {
+  limit?: number;
+  offset?: number;
+}
+interface DatasetListOptions extends PageOptions {
+  search?: string;
+  archived?: boolean;
+  orderBy?: DatasetOrder;
+  tags?: readonly string[];
+  tagMatch?: 'any' | 'all';
+}
+interface ItemListOptions extends PageOptions {
+  revision?: number;
+  archived?: boolean;
+}
+interface CreateDatasetItemOptions {
+  input: Exclude<JSONValue, null>;
+  expectedOutput?: JSONValue;
+  sourceOutput?: JSONValue;
+  metadata?: JSONObject;
+  clientItemId?: string | null;
+  sourceTraceId?: string | null;
+  sourceEventId?: string | null;
+  sourceTimestamp?: string | Date | null;
+}
+interface UpdateDatasetItemOptions {
+  baseVersion: number;
+  input?: Exclude<JSONValue, null>;
+  expectedOutput?: JSONValue;
+  metadata?: JSONObject;
+}
+/** Dataset management. Writes require a personal key with dataset:write. */
+declare class Datasets {
+  #private;
+  constructor(transport: Transport);
+  create(options: DatasetMetadata & {
+    name: string;
+  }, request?: RequestOptions): Promise<Dataset>;
+  get(datasetId: string, request?: RequestOptions): Promise<Dataset>;
+  list(options?: DatasetListOptions): Promise<DatasetPage>;
+  update(datasetId: string, options: DatasetMetadata, request?: RequestOptions): Promise<Dataset>;
+  archive(datasetId: string, request?: RequestOptions): Promise<Dataset>;
+  restore(datasetId: string, request?: RequestOptions): Promise<Dataset>;
+  listRevisions(datasetId: string, options?: PageOptions): Promise<DatasetRevisionPage>;
+  listItems(datasetId: string, options?: ItemListOptions): Promise<DatasetItemPage>;
+  getItem(itemId: string, options?: RequestOptions & {
+    revision?: number;
+  }): Promise<DatasetItem>;
+  createItem(datasetId: string, options: CreateDatasetItemOptions, request?: RequestOptions): Promise<DatasetItem>;
+  updateItem(itemId: string, options: UpdateDatasetItemOptions, request?: RequestOptions): Promise<DatasetItem>;
+  archiveItem(itemId: string, options: {
+    baseVersion: number;
+  }, request?: RequestOptions): Promise<DatasetItem>;
+  restoreItem(itemId: string, options: {
+    baseVersion: number;
+    sourceVersion?: number;
+  }, request?: RequestOptions): Promise<DatasetItem>;
+  listItemVersions(itemId: string, options?: PageOptions): Promise<DatasetItemPage>;
+  snapshot(datasetId: string, options?: RequestOptions & {
+    revision?: number;
+  }): Promise<DatasetSnapshot>;
+}
+/** A repeatable, lazy read of active items at one immutable revision. Keep its client open. */
+declare class DatasetSnapshot implements AsyncIterable<DatasetItem> {
+  #private;
+  readonly revision: number;
+  readonly revisionId: string;
+  constructor(datasets: Datasets, dataset: Dataset, revisionNumber: number, revisionId: string);
+  get dataset(): Dataset;
+  iterItems(options?: RequestOptions & {
+    pageSize?: number;
+  }): AsyncIterableIterator<DatasetItem>;
+  [Symbol.asyncIterator](): AsyncIterableIterator<DatasetItem>;
+}
+//#endregion
+//#region src/evaluations/declarations.d.ts
+interface ItemPayload {
+  input?: JSONValue;
+  output?: JSONValue;
+  expectedOutput?: JSONValue;
+  metadata?: JSONObject | null;
+}
+interface ResultPayload {
+  reasoning?: string | null;
+  errorMessage?: string | null;
+  metadata?: JSONObject | null;
+}
+type PayloadOptions<T> = (T & {
+  payload?: never;
+}) | ({
+  payload: T;
+} & { [K in keyof T]?: never; });
+interface ItemIdentifiers {
+  id?: string;
+  caseKey?: string | null;
+  trial?: string | null;
+  datasetItemIdentifier?: string | null;
+  datasetItemVersionIdentifier?: string | null;
+  datasetItemVersionId?: string | null;
+  applicationTraceId?: string | null;
+}
+type EvaluationItemOptions = ItemIdentifiers & PayloadOptions<ItemPayload>;
+type ScoreValue = boolean | number | readonly string[];
+/** bigint inputs are converted only if exactly representable as finite binary64 scores. */
+type ScoreInput = ScoreValue | bigint;
+type ResultStatus = 'ok' | 'error' | 'skipped' | 'not_applicable';
+type Outcome = ({
+  status?: 'ok';
+  value: ScoreInput;
+  errorCode?: never;
+} & PayloadOptions<Omit<ResultPayload, 'errorMessage'> & {
+  errorMessage?: null;
+}>) | ({
+  status: 'error';
+  value?: never;
+  errorCode?: string;
+} & PayloadOptions<ResultPayload>) | ({
+  status: 'skipped' | 'not_applicable';
+  value?: never;
+  errorCode?: never;
+} & PayloadOptions<Omit<ResultPayload, 'errorMessage'> & {
+  errorMessage?: null;
+}>);
+type EvaluationResultOptions = Outcome & {
+  item: EvaluationItem | string;
+  scorerVersionId: string;
+  evaluatorTraceId?: string | null;
+  evaluatedAt?: string | Date | null;
+};
+/** One immutable execution. Reuse across scorers; construct a new item for each trial. */
+declare class EvaluationItem {
+  readonly id: string;
+  constructor(options?: EvaluationItemOptions);
+  /** Detached wire-format object. Use toJSONString() for lossless cross-language persistence. */
+  toJSON(): WireObject;
+  /** Exact wire JSON, including large integers. Restore with fromJSON(JSON.parse(saved)). */
+  toJSONString(): string;
+  static fromJSON(value: unknown): EvaluationItem;
+  static fromDatasetItem(item: DatasetItem, options?: {
+    output?: JSONValue;
+    id?: string;
+    caseKey?: string;
+    trial?: string;
+    applicationTraceId?: string;
+  }): EvaluationItem;
+}
+/** An evaluator outcome pinned to an exact scorer version. `ok` does not imply passing. */
+declare class EvaluationResult {
+  readonly item: EvaluationItem | string;
+  readonly scorerVersionId: string;
+  readonly itemId: string;
+  constructor(options: EvaluationResultOptions);
+  /** Detached wire-format object. Use toJSONString() for lossless cross-language persistence. */
+  toJSON(): WireObject;
+  /** Exact wire JSON. Save the attached item separately and restore with fromJSON(JSON.parse(saved), { item }). */
+  toJSONString(): string;
+  static fromJSON(value: unknown, options?: {
+    item?: EvaluationItem;
+  }): EvaluationResult;
+}
+//#endregion
+//#region src/evaluations/errors.d.ts
+type Persistence = 'not_sent' | 'rejected' | 'unknown';
+interface EvaluationErrorFields {
+  status?: number;
+  code?: string;
+  detail?: unknown;
+  attr?: string;
+  errors?: unknown;
+  response?: WireObject;
+  /** Retry-After in seconds. */
+  retryAfter?: number;
+  persistence?: Persistence;
+  submission?: ExperimentOptions;
+}
+/** Native cancellation with the prepared creation fields needed for recovery. */
+declare class ExperimentCreationAbortedError extends DOMException {
+  #private;
+  readonly cause: unknown;
+  constructor(cause: unknown, submission: ExperimentOptions);
+  get submission(): ExperimentOptions;
+}
+/** An operation did not receive a valid acknowledgment. Inspect persistence before recovery. */
+declare class EvaluationAPIError extends Error {
+  #private;
+  readonly errors: unknown;
+  readonly status?: number;
+  readonly code?: string;
+  readonly detail?: unknown;
+  readonly attr?: string;
+  readonly response?: WireObject;
+  readonly retryAfter?: number;
+  persistence: Persistence;
+  constructor(fields?: EvaluationErrorFields);
+  get submission(): ExperimentOptions | undefined;
+}
+//#endregion
+//#region src/evaluations/scorers.d.ts
+interface BooleanScorerConfig {
+  trueLabel?: string;
+  falseLabel?: string;
+  trueIsFailure?: boolean | null;
+}
+interface NumericPassingRule {
+  operator: 'gte' | 'lte';
+  threshold: number;
+}
+interface NumericScorerConfig {
+  min?: number | null;
+  max?: number | null;
+  step?: number | null;
+  passingRule?: NumericPassingRule | null;
+}
+interface CategoricalScorerOption {
+  key: string;
+  label: string;
+}
+interface CategoricalScorerConfig {
+  options: readonly CategoricalScorerOption[];
+  selectionMode?: 'single' | 'multiple';
+  minSelections?: number | null;
+  maxSelections?: number | null;
+  passingRule?: {
+    categories: readonly string[];
+  } | null;
+}
+interface ScorerConfigs {
+  boolean: BooleanScorerConfig;
+  numeric: NumericScorerConfig;
+  categorical: CategoricalScorerConfig;
+}
+type ScorerKind = keyof ScorerConfigs;
+type ScorerConfig = ScorerConfigs[ScorerKind];
+interface ScorerRecord {
+  id: string;
+  name: string;
+  description: string;
+  archived: boolean;
+  tags: string[];
+  currentVersion: number;
+  currentVersionId: string;
+  team: number;
+  createdAt: string;
+  updatedAt: string;
+  createdBy: JSONObject | null;
+}
+interface VersionRecord {
+  id: string;
+  definitionId: string;
+  version: number;
+  createdAt: string;
+  createdBy: JSONObject | null;
+}
+type Scorer = { [K in ScorerKind]: ScorerRecord & {
+  kind: K;
+  config: ScorerConfigs[K];
+}; }[ScorerKind];
+type ScorerVersion = { [K in ScorerKind]: VersionRecord & {
+  kind: K;
+  config: ScorerConfigs[K];
+}; }[ScorerKind];
+interface ScorerPage extends Page<Scorer> {
+  next: string | null;
+  previous: string | null;
+}
+interface ScorerVersionPage {
+  count: EvaluationCount;
+  results: ScorerVersion[];
+  nextCursor: string | null;
+}
+type ScorerOrder = 'name' | '-name' | 'kind' | '-kind' | 'created_at' | '-created_at' | 'updated_at' | '-updated_at' | 'current_version' | '-current_version';
+interface ScorerListOptions extends PageOptions {
+  search?: string;
+  kind?: ScorerKind;
+  archived?: boolean;
+  orderBy?: ScorerOrder;
+  tags?: readonly string[];
+  tagMatch?: 'any' | 'all';
+}
+interface ScorerMetadata {
+  name?: string;
+  description?: string | null;
+  archived?: boolean;
+  tags?: readonly string[];
+}
+type CreateScorerOptions = { [K in ScorerKind]: {
+  name: string;
+  kind: K;
+  config: ScorerConfigs[K];
+  description?: string | null;
+  tags?: readonly string[];
+}; }[ScorerKind];
+/** Versioned scorer definitions. Evaluator execution remains caller-owned. */
+declare class Scorers {
+  #private;
+  constructor(transport: Transport);
+  create<T extends CreateScorerOptions>(options: T, request?: RequestOptions): Promise<Extract<Scorer, {
+    kind: T['kind'];
+  }>>;
+  get(scorerId: string, request?: RequestOptions): Promise<Scorer>;
+  list(options?: ScorerListOptions): Promise<ScorerPage>;
+  update(scorerId: string, options: ScorerMetadata, request?: RequestOptions): Promise<Scorer>;
+  createVersion(scorerId: string, options: {
+    config: ScorerConfig;
+    baseVersion?: number;
+    name?: string;
+    description?: string | null;
+  }, request?: RequestOptions): Promise<Scorer>;
+  getVersion(scorerId: string, versionId: string, request?: RequestOptions): Promise<ScorerVersion>;
+  listVersions(scorerId: string, options?: RequestOptions & {
+    limit?: number;
+    cursor?: string;
+  }): Promise<ScorerVersionPage>;
+}
+//#endregion
+//#region src/evaluations/client.d.ts
+interface ExperimentOptions {
+  name: string;
+  id?: string;
+  startedAt?: string | Date;
+  runSource?: 'ci' | 'local' | 'scheduled' | null;
+  expectedItemCount?: EvaluationCount | null;
+  expectedResultCount?: EvaluationCount | null;
+  suiteKey?: string | null;
+  datasetSource?: string | null;
+  datasetIdentifier?: string | null;
+  datasetRevisionIdentifier?: string | null;
+  datasetRevisionId?: string | null;
+  applicationVersion?: string | null;
+  modelVersion?: string | null;
+  promptVersion?: string | null;
+}
+interface ExperimentReceipt {
+  id: string;
+  status: 'uploading' | 'completed' | 'failed';
+  created: boolean;
+  startedAt: string;
+  createdAt: string;
+  finishedAt: string | null;
+  expectedItemCount: EvaluationCount | null;
+  expectedResultCount: EvaluationCount | null;
+  acceptedItemCount: EvaluationCount;
+  acceptedResultCount: EvaluationCount;
+}
+interface ItemReceipt {
+  id: string;
+  created: boolean;
+  acceptedAt: string;
+}
+interface ResultReceipt extends ItemReceipt {
+  itemId: string;
+  scorerVersionId: string;
+}
+interface UploadReceipt {
+  items: ItemReceipt[];
+  results: ResultReceipt[];
+}
+interface BulkUploadReceipt {
+  chunks: UploadReceipt[];
+}
+interface Chunk {
+  start: number;
+  results: EvaluationResult[];
+  body: string;
+}
+/** Earlier chunks are acknowledged; pendingResults includes the uncertain failed chunk. */
+declare class BulkUploadError extends EvaluationAPIError {
+  #private;
+  readonly failedIndex: number;
+  constructor(cause: EvaluationAPIError, completed: BulkUploadReceipt, failed: Chunk, pending: readonly EvaluationResult[]);
+  get completed(): BulkUploadReceipt;
+  get pendingResults(): readonly EvaluationResult[];
+  get failedChunk(): WireObject;
+}
+declare class Experiment {
+  #private;
+  readonly id: string;
+  constructor(transport: Transport, id: string, receipt?: ExperimentReceipt, submission?: string);
+  get receipt(): ExperimentReceipt | undefined;
+  /** Original creation options, suitable for createExperiment(submission). Large counts may be bigint. */
+  get submission(): ExperimentOptions | undefined;
+  uploadResult(options: EvaluationResultOptions, request?: RequestOptions): Promise<UploadReceipt>;
+  uploadResults(results: readonly EvaluationResult[], request?: RequestOptions): Promise<BulkUploadReceipt>;
+  complete(request?: RequestOptions): Promise<ExperimentReceipt>;
+  fail(request?: RequestOptions): Promise<ExperimentReceipt>;
+  getTags(request?: RequestOptions): Promise<string[]>;
+  setTags(names: readonly string[], request?: RequestOptions): Promise<string[]>;
+}
+/** Acknowledged management APIs, independent of analytics capture and flush. */
+declare class OfflineEvaluations {
+  #private;
+  readonly datasets: Datasets;
+  readonly scorers: Scorers;
+  constructor(options: OfflineEvaluationsOptions);
+  createExperiment(options: ExperimentOptions, request?: RequestOptions): Promise<Experiment>;
+  /** Local only; existence and permissions are checked on the next network operation. */
+  resumeExperiment(id: string): Experiment;
+  /** Stop owned requests and timers. This never completes or fails experiments. */
+  close(): void;
+}
+//#endregion
+export { type BooleanScorerConfig, BulkUploadError, type BulkUploadReceipt, type CategoricalScorerConfig, type CategoricalScorerOption, type CreateDatasetItemOptions, type CreateScorerOptions, type Dataset, type DatasetItem, type DatasetItemPage, type DatasetListOptions, type DatasetMetadata, type DatasetOrder, type DatasetPage, type DatasetRevision, type DatasetRevisionPage, DatasetSnapshot, Datasets, EvaluationAPIError, type EvaluationCount, type EvaluationErrorFields, EvaluationItem, type EvaluationItemOptions, EvaluationResult, type EvaluationResultOptions, Experiment, ExperimentCreationAbortedError, type ExperimentOptions, type ExperimentReceipt, type ItemIdentifiers, type ItemListOptions, type ItemPayload, type ItemReceipt, type JSONObject, type JSONValue, type NumericPassingRule, type NumericScorerConfig, OfflineEvaluations, type OfflineEvaluationsOptions, type PageOptions, type Persistence, type RequestOptions, type ResultPayload, type ResultReceipt, type ResultStatus, type ScoreInput, type ScoreValue, type Scorer, type ScorerConfig, type ScorerConfigs, type ScorerKind, type ScorerListOptions, type ScorerMetadata, type ScorerOrder, type ScorerPage, type ScorerVersion, type ScorerVersionPage, Scorers, type UpdateDatasetItemOptions, type UploadReceipt };
+//# sourceMappingURL=index.d.ts.map