@templatical/ai-kit 0.0.0-stage → 0.51.0

package/dist/testing/index.d.ts

@@ -0,0 +1,277 @@
+import { t as AiModelAdapter } from "../model-xJvGTfLB.js";
+import { t as HttpAiFeatures } from "../http-provider-Cb41NXd5.js";
+import { C as AiWireEvent } from "../wire-Cv7Fk-7n.js";
+import { AiAltTextRequest, AiAppliedBatch, AiChatAttachment, AiChatRequest, AiFixFinding, AiFixRequest, AiMergeTags, AiOperationsResult, AiProvider, AiRewriteRequest, AiTurnHandlers, Operation, TemplateContent } from "@templatical/types";
+//#region src/testing/conformance.d.ts
+interface AiConformanceCheck {
+  name: string;
+  ok: boolean;
+  detail: string;
+}
+interface AiConformanceReport {
+  ok: boolean;
+  checks: AiConformanceCheck[];
+}
+interface AiConformanceOptions {
+  /** The template the suite edits; the test set's welcome email by default. */
+  content?: TemplateContent;
+  templateId?: string;
+  /** Public probe image URL; defaults to AI_CONFORMANCE_ALT_TEXT_IMAGE_URL. */
+  altTextImageUrl?: string;
+  mergeTags?: AiMergeTags;
+  /** How long a stopped call may take to reject (D35). */
+  timeoutMs?: number;
+  /** An optional shorter call limit; every call is also bounded by `timeoutMs`. */
+  callTimeoutMs?: number;
+}
+/**
+ * Drive a provider as the editor would and check the contract (A1 §6): any
+ * test runner, never throws on a failed check. Each declared feature runs
+ * once against a stand-in editor that applies `operations` with the engine.
+ */
+export declare function runAiProviderConformance(provider: AiProvider, options?: AiConformanceOptions): Promise<AiConformanceReport>;
+//#endregion
+//#region src/testing/wire-conformance.d.ts
+interface AiWireConformanceOptions {
+  baseUrl: string;
+  features?: HttpAiFeatures;
+  headers?: Record<string, string> | (() => Record<string, string> | Promise<Record<string, string>>);
+  templateId?: string;
+  /** Defaults to AI_CONFORMANCE_ALT_TEXT_IMAGE_URL. */
+  altTextImageUrl?: string;
+  timeoutMs?: number;
+  callTimeoutMs?: number;
+}
+/**
+ * The provider suite over the wire, through `createHttpAiProvider`, plus the
+ * wire's own rules: every response carries `Templatical-AI-Wire: 1`, streams
+ * are `text/event-stream`, and a bad turn is refused before the stream with a
+ * JSON error. For a backend in any language; run it from Node (no CORS).
+ */
+export declare function runAiWireConformance(options: AiWireConformanceOptions): Promise<AiConformanceReport>;
+//#endregion
+//#region src/testing/mock-provider.d.ts
+/** A wire event to play, or a pause. */
+type AiRecordedEvent = AiWireEvent | {
+  delayMs: number;
+};
+/** Events, or events built from the request (to name its blocks). */
+type AiRecording<R> = AiRecordedEvent[] | ((request: R) => AiRecordedEvent[]);
+interface AiRecordedTurn {
+  /** The first turn whose `match` fits the message plays; one without `match` always fits. */
+  match?: RegExp;
+  events: AiRecording<AiChatRequest>;
+}
+/** What the mock plays. A missing feature is off (`false`). */
+interface AiMockScript {
+  chat?: false | AiRecordedTurn[];
+  fixFinding?: false | AiRecording<AiFixRequest>;
+  chatAttachments?: boolean;
+  rewrite?: false | AiRecording<AiRewriteRequest>;
+  altText?: false | AiRecording<AiAltTextRequest>;
+  /** Keep a conversation in memory per template, with Undo and Redo. */
+  conversation?: boolean;
+  /** A pause before every event, in ms (0 by default). */
+  delayMs?: number;
+}
+/**
+ * An `AiProvider` that plays recorded streams: for integrators' UI tests and
+ * the playground's AI scene (no model, no keys). Turns apply their
+ * `operations` through the editor's handlers like a real backend's; with
+ * `conversation: true` it keeps history in memory, Undo answering the turn's
+ * inverse and Redo the guarded inverse of Undo. Replays change history only
+ * after settlement.
+ */
+export declare function createMockAiProvider(script: AiMockScript): AiProvider;
+//#endregion
+//#region src/testing/recordings.d.ts
+/**
+ * Scripted turns for any template: the playground's AI scene and the
+ * conformance suite's self-test play them. Chat answers "dark", "title" and
+ * "fail" specially and otherwise adds a closing paragraph.
+ */
+export declare const AI_RECORDINGS: AiMockScript;
+//#endregion
+//#region src/testing/eval-set.d.ts
+type AiEvalCheck = {
+  type: "applied" | "notApplied";
+} | {
+  type: "present" | "missing";
+  block: string;
+} | {
+  type: "contains" | "notContains";
+  block?: string;
+  text: string;
+} | {
+  type: "lintGone";
+  ruleId: string;
+} | {
+  type: "field" | "fieldNot";
+  block: string;
+  path: string;
+  matches: string;
+} | {
+  type: "setting";
+  path: string;
+  matches: string;
+} | {
+  type: "greater";
+  block: string;
+  path: string;
+  than: number;
+} | {
+  type: "maxText";
+  block: string;
+  chars: number;
+} | {
+  type: "countType";
+  blockType: string;
+  min: number;
+} | {
+  type: "order";
+  first: string;
+  then: string;
+} | {
+  type: "firstContains";
+  text: string;
+} | {
+  type: "reply";
+  matches: string;
+};
+type AiEvalAltCheck = {
+  type: "nonEmpty" | "withinLimit";
+} | {
+  type: "containsAll" | "containsAny";
+  texts: string[];
+} | {
+  type: "notMatches";
+  matches: string;
+} | {
+  type: "shortOrMatches";
+  chars: number;
+  matches: string;
+};
+type EvalTemplate = "welcome" | "promo" | "injection" | "empty" | "domain-mismatch" | "low-text";
+interface EvalTurnCase {
+  name: string;
+  template: EvalTemplate;
+  /** Block ids the case may delete; every other original id must survive. */
+  removed?: string[];
+  checks: AiEvalCheck[];
+}
+interface AiEvalAltTextInput {
+  /** Path relative to the package's eval/ folder. */
+  image: string;
+  linkHref: string | null;
+  nearbyText: string;
+  maxLength: number;
+}
+type AiEvalCase = (EvalTurnCase & {
+  kind?: "edit";
+  message: string;
+  selectedBlockId?: string;
+  attachments?: {
+    file: string;
+    kind: "image" | "pdf";
+  }[];
+}) | (EvalTurnCase & {
+  kind: "fix";
+  finding: AiFixFinding;
+}) | (AiEvalAltTextInput & {
+  kind: "altText";
+  name: string;
+  checks: AiEvalAltCheck[];
+});
+/** The cases of S7's test set (`eval/cases.json`). An omitted kind means edit. */
+export declare const AI_EVAL_CASES: AiEvalCase[];
+export declare const AI_EVAL_TEMPLATES: Readonly<Record<EvalTemplate, TemplateContent>>;
+/** The merge tags every case's editor declares. */
+export declare const AI_EVAL_MERGE_TAGS: AiMergeTags;
+interface AiEvalTurnInput {
+  content: TemplateContent;
+  mergeTags: AiMergeTags;
+  message: string;
+  selectedBlockId: string | null;
+  finding: AiFixFinding | null;
+  attachments: AiChatAttachment[];
+}
+interface AiEvalTurnOutput {
+  /** The template after the turn. */
+  content: TemplateContent;
+  reply: string;
+  /** Whether the turn changed the template. */
+  applied: boolean;
+}
+interface AiEvalReport {
+  model: string;
+  passed: number;
+  /** Cases that ran; skipped cases are excluded. */
+  total: number;
+  skipped: number;
+  /** Whole percent, rounded down. */
+  rate: number;
+  ok: boolean;
+  results: {
+    name: string;
+    ok: boolean;
+    reason: string | null;
+    skipped?: true;
+  }[];
+}
+/**
+ * Run one case's checks against a turn's outcome; the first failure's reason,
+ * or `null` when it passed (S7 Cloud Task 11's `failure()`).
+ */
+export declare function checkEvalCase(testCase: Exclude<AiEvalCase, {
+  kind: "altText";
+}>, original: TemplateContent, output: AiEvalTurnOutput, lintCheck?: (content: TemplateContent) => string[]): string | null;
+/** Text-free public probe image from this package's fixed-version SDK release. */
+export declare const AI_CONFORMANCE_ALT_TEXT_IMAGE_URL: string;
+/**
+ * Run each case on a fresh template. Missing optional runners skip their cases;
+ * thrown turns or checks fail. The pass rate counts only cases that ran.
+ */
+export declare function runAiEvalSet(options: {
+  runTurn: (input: AiEvalTurnInput) => Promise<AiEvalTurnOutput>;
+  runAltText?: (input: AiEvalAltTextInput) => Promise<string>;
+  lintCheck?: (content: TemplateContent) => string[];
+  loadFile?: (file: string) => Promise<Blob>;
+  model: string;
+  cases?: readonly AiEvalCase[];
+  minPassRate?: number;
+}): Promise<AiEvalReport>;
+/** A `runAltText` using image URLs; override them for unreleased fixtures. */
+export declare function evalAltTextWith(model: AiModelAdapter, options?: {
+  imageUrl?: (file: string) => string;
+}): (input: AiEvalAltTextInput) => Promise<string>;
+/** A `runTurn` on the kit's own loop, for a model behind an `AiModelAdapter`. */
+export declare function evalTurnWith(model: AiModelAdapter, options?: {
+  maxSteps?: number;
+}): (input: AiEvalTurnInput) => Promise<AiEvalTurnOutput>;
+//#endregion
+//#region src/testing/stand-in-editor.d.ts
+/** The editor the suites drive: `onOperations` applies with the engine. */
+interface StandInEditor {
+  content: TemplateContent;
+  readonly mergeTags: AiMergeTags;
+  readonly sessionId: string;
+  readonly applied: Operation[][];
+  readonly refused: {
+    code: string;
+    opIndex: number;
+    message: string;
+  }[];
+  readonly serverBatches: AiAppliedBatch[];
+  /** Apply a batch as the editor would answer `onOperations`. */
+  apply(ops: Operation[]): AiOperationsResult;
+  /** Fresh handlers for one call; `text` collects what streamed. */
+  handlers(extra?: {
+    onText?: (text: string) => void;
+  }): AiTurnHandlers & {
+    readonly text: string;
+  };
+}
+export declare function createStandInEditor(content: TemplateContent, mergeTags?: AiMergeTags): StandInEditor;
+//#endregion
+export type { AiConformanceCheck, AiConformanceOptions, AiConformanceReport, AiEvalAltCheck, AiEvalAltTextInput, AiEvalCase, AiEvalCheck, AiEvalReport, AiEvalTurnInput, AiEvalTurnOutput, AiMockScript, AiRecordedEvent, AiRecordedTurn, AiRecording, AiWireConformanceOptions, StandInEditor };
+//# sourceMappingURL=index.d.ts.map