@sanity/ailf 0.1.5 → 0.1.7

package/dist/_vendor/ailf-tasks/schemas.js

@@ -0,0 +1,176 @@
+/**
+ * schemas.ts — Zod schemas for repo-based task definitions.
+ *
+ * Validates .ailf/tasks/*.yaml task files from external repositories.
+ * These schemas are the contract between external repos and the AILF eval
+ * pipeline — they define exactly what fields are accepted, with friendly
+ * error messages for authors writing task YAML by hand.
+ *
+ * This module is the single source of truth for task schemas. The eval
+ * package re-exports from here to avoid duplication.
+ *
+ * @see docs/exec-plans/completed/tasks-as-content/phase-4-repo-based-tasks.md
+ */
+import { z } from "zod";
+// ---------------------------------------------------------------------------
+// Constants — curated assertion types and rubric template names
+// ---------------------------------------------------------------------------
+/**
+ * The set of assertion types allowed in repo-based task files.
+ *
+ * This is a curated subset of Promptfoo assertion types — we expose only the
+ * types that are stable, well-documented, and useful for external authors.
+ */
+export const CURATED_ASSERTION_TYPES = [
+    "llm-rubric",
+    "contains",
+    "contains-any",
+    "contains-all",
+    "not-contains",
+    "icontains",
+    "icontains-any",
+    "regex",
+    "javascript",
+    "similar",
+    "cost",
+    "latency",
+];
+/**
+ * Valid rubric template names — must match keys in config/rubrics.yaml.
+ */
+export const RUBRIC_TEMPLATE_NAMES = [
+    "task-completion",
+    "code-correctness",
+    "doc-coverage",
+];
+// ---------------------------------------------------------------------------
+// Assertion schemas
+// ---------------------------------------------------------------------------
+/**
+ * Polymorphic canonical doc reference — discriminated by key presence.
+ * Exactly one resolution key (slug, path, id, or perspective) must be present.
+ *
+ * @see docs/design-docs/canonical-doc-resolution.md
+ */
+const SlugDocRefSchema = z.object({
+    slug: z.string().min(1),
+    reason: z.string().optional().default(""),
+});
+const PathDocRefSchema = z.object({
+    path: z.string().min(1),
+    reason: z.string().optional().default(""),
+});
+const IdDocRefSchema = z.object({
+    id: z.string().min(1),
+    reason: z.string().optional().default(""),
+    /** Human-readable slug annotation (not used for resolution) */
+    slug: z.string().optional(),
+    /** Human-readable path annotation (not used for resolution) */
+    path: z.string().optional(),
+});
+const PerspectiveDocRefSchema = z.object({
+    perspective: z.string().min(1),
+    reason: z.string().optional().default(""),
+});
+// Order matters: IdDocRefSchema first because it may also carry `slug`
+// and `path` as optional annotations. Zod tries schemas in order, so
+// entries like `{ id: "...", slug: "..." }` must match IdDocRefSchema
+// (not SlugDocRefSchema).
+const CanonicalDocRefSchema = z.union([
+    IdDocRefSchema,
+    SlugDocRefSchema,
+    PathDocRefSchema,
+    PerspectiveDocRefSchema,
+]);
+/**
+ * A templated LLM-rubric assertion — uses one of the predefined rubric
+ * templates with author-supplied criteria.
+ */
+const TemplatedAssertionSchema = z.object({
+    type: z.literal("llm-rubric"),
+    template: z.enum(RUBRIC_TEMPLATE_NAMES),
+    criteria: z.array(z.string().min(1)).min(1),
+    weight: z.number().optional(),
+});
+/**
+ * A value-based assertion (contains, regex, cost, etc.). Uses .passthrough()
+ * to allow extra fields for future extension without schema breakage.
+ */
+const ValueAssertionSchema = z
+    .object({
+    type: z.enum(CURATED_ASSERTION_TYPES),
+    value: z.unknown().optional(),
+    threshold: z.number().optional(),
+    weight: z.number().optional(),
+})
+    .passthrough();
+/** Union of all supported assertion shapes. */
+const AssertionSchema = z.union([
+    TemplatedAssertionSchema,
+    ValueAssertionSchema,
+]);
+// ---------------------------------------------------------------------------
+// Nested config schemas
+// ---------------------------------------------------------------------------
+const BaselineConfigSchema = z
+    .object({
+    enabled: z.boolean().optional(),
+    rubric: z.enum(["abbreviated", "full", "none"]).optional(),
+})
+    .optional();
+const ExecutionConfigSchema = z
+    .object({
+    enabled: z.boolean().optional().default(true),
+    blocking: z.boolean().optional().default(false),
+    threshold: z
+        .object({
+        score: z.number().min(0).max(100).optional(),
+    })
+        .optional(),
+    trigger: z
+        .object({
+        branches: z.array(z.string()).optional(),
+        paths: z.array(z.string()).optional(),
+    })
+        .optional(),
+    source: z.string().optional(),
+})
+    .optional();
+// ---------------------------------------------------------------------------
+// RepoTaskSchema — a single task definition from .ailf/tasks/*.yaml
+// ---------------------------------------------------------------------------
+/**
+ * Zod schema for a single repo-based task definition.
+ *
+ * This is the external-author-facing contract. Field names are camelCase
+ * to match the Content Lake document schema (ailf.task).
+ */
+export const RepoTaskSchema = z.object({
+    id: z
+        .string()
+        .min(1)
+        .regex(/^[a-z0-9][a-z0-9-]*$/, "Task ID must be lowercase alphanumeric with hyphens"),
+    description: z.string().min(1),
+    featureArea: z
+        .string()
+        .min(1)
+        .regex(/^[a-z0-9][a-z0-9-]*$/, "Feature area must be lowercase alphanumeric with hyphens"),
+    tags: z.array(z.string()).optional(),
+    canonicalDocs: z.array(CanonicalDocRefSchema).optional().default([]),
+    vars: z
+        .object({
+        task: z.string().min(1),
+    })
+        .passthrough()
+        .optional(),
+    assert: z.array(AssertionSchema).min(1),
+    baseline: BaselineConfigSchema,
+    docCoverage: z.boolean().optional().default(false),
+    referenceSolution: z.string().optional(),
+    execution: ExecutionConfigSchema,
+});
+/**
+ * Schema for an array of repo tasks — what a single .ailf/tasks/*.yaml file
+ * contains. Each file must define at least one task.
+ */
+export const RepoTaskFileSchema = z.array(RepoTaskSchema).min(1);

package/dist/_vendor/ailf-tasks/validation.d.ts

@@ -0,0 +1,47 @@
+/**
+ * validation.ts — Semantic validation for repo-based tasks.
+ *
+ * Checks that go beyond Zod schema parsing:
+ * - Assertion types are in the curated set
+ * - Rubric template names resolve to known templates
+ * - Feature area strings are well-formed
+ * - Canonical doc slugs look reasonable (slugs, not URLs)
+ *
+ * These produce warnings, not errors — the pipeline can still run
+ * with imperfect tasks. Only structural failures (caught by Zod) block.
+ */
+import { type RepoTask } from "./schemas.js";
+export interface ValidationResult {
+    valid: boolean;
+    errors: ValidationMessage[];
+    warnings: ValidationMessage[];
+}
+export interface ValidationMessage {
+    taskId: string;
+    field: string;
+    message: string;
+}
+/**
+ * Run semantic validation on an array of parsed repo tasks.
+ *
+ * Returns warnings for issues that don't block execution (unknown feature
+ * areas, unresolved slugs) and errors for issues that would cause pipeline
+ * failures (completely missing required fields — though Zod catches most).
+ */
+export declare function validateRepoTasks(tasks: RepoTask[]): ValidationResult;
+/**
+ * Format validation results for console output.
+ */
+export declare function formatValidationResult(result: ValidationResult): string;
+/**
+ * Detect snake_case field names in raw task YAML data.
+ *
+ * This runs BEFORE Zod parsing to provide a user-friendly error message
+ * when authors use framework-internal snake_case names instead of the
+ * camelCase names expected in repo task files.
+ *
+ * @param raw - Raw parsed YAML (before Zod validation)
+ * @param filename - Source filename for error messages
+ * @returns Array of warning messages (empty if no issues)
+ */
+export declare function detectSnakeCaseFields(raw: unknown, filename: string): string[];

package/dist/_vendor/ailf-tasks/validation.js

@@ -0,0 +1,162 @@
+/**
+ * validation.ts — Semantic validation for repo-based tasks.
+ *
+ * Checks that go beyond Zod schema parsing:
+ * - Assertion types are in the curated set
+ * - Rubric template names resolve to known templates
+ * - Feature area strings are well-formed
+ * - Canonical doc slugs look reasonable (slugs, not URLs)
+ *
+ * These produce warnings, not errors — the pipeline can still run
+ * with imperfect tasks. Only structural failures (caught by Zod) block.
+ */
+import { CURATED_ASSERTION_TYPES, RUBRIC_TEMPLATE_NAMES, } from "./schemas.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Run semantic validation on an array of parsed repo tasks.
+ *
+ * Returns warnings for issues that don't block execution (unknown feature
+ * areas, unresolved slugs) and errors for issues that would cause pipeline
+ * failures (completely missing required fields — though Zod catches most).
+ */
+export function validateRepoTasks(tasks) {
+    const errors = [];
+    const warnings = [];
+    // Check for duplicate IDs
+    const seenIds = new Set();
+    for (const task of tasks) {
+        if (seenIds.has(task.id)) {
+            errors.push({
+                taskId: task.id,
+                field: "id",
+                message: `Duplicate task ID "${task.id}"`,
+            });
+        }
+        seenIds.add(task.id);
+    }
+    for (const task of tasks) {
+        // Check assertion types
+        for (let i = 0; i < task.assert.length; i++) {
+            const assertion = task.assert[i];
+            if (!CURATED_ASSERTION_TYPES.includes(assertion.type)) {
+                warnings.push({
+                    taskId: task.id,
+                    field: `assert[${i}].type`,
+                    message: `Unknown assertion type "${assertion.type}". ` +
+                        `Valid types: ${CURATED_ASSERTION_TYPES.join(", ")}`,
+                });
+            }
+            // Check rubric template for llm-rubric assertions
+            if (assertion.type === "llm-rubric" && "template" in assertion) {
+                const template = assertion.template;
+                if (!RUBRIC_TEMPLATE_NAMES.includes(template)) {
+                    warnings.push({
+                        taskId: task.id,
+                        field: `assert[${i}].template`,
+                        message: `Unknown rubric template "${template}". ` +
+                            `Valid templates: ${RUBRIC_TEMPLATE_NAMES.join(", ")}`,
+                    });
+                }
+            }
+        }
+        // Check canonical doc refs look reasonable
+        for (let i = 0; i < (task.canonicalDocs?.length ?? 0); i++) {
+            const doc = task.canonicalDocs[i];
+            // Slug refs: warn if they look like URLs or paths
+            if ("slug" in doc && !("id" in doc) && typeof doc.slug === "string") {
+                if (doc.slug.includes("/") || doc.slug.includes("http")) {
+                    warnings.push({
+                        taskId: task.id,
+                        field: `canonicalDocs[${i}].slug`,
+                        message: `Slug "${doc.slug}" looks like a URL or path — use 'path' type for paths or 'slug' for document slugs (e.g., "groq-introduction")`,
+                    });
+                }
+            }
+        }
+        // Check task has at least one llm-rubric assertion (recommended but not required)
+        const hasLlmRubric = task.assert.some((a) => a.type === "llm-rubric");
+        if (!hasLlmRubric) {
+            warnings.push({
+                taskId: task.id,
+                field: "assert",
+                message: "No llm-rubric assertion found. Tasks should have at least one scored rubric for meaningful evaluation.",
+            });
+        }
+        // Check taskPrompt exists in vars (vars.task)
+        if (!task.vars?.task) {
+            warnings.push({
+                taskId: task.id,
+                field: "vars.task",
+                message: "No task prompt found in vars.task. The LLM will receive an empty implementation request.",
+            });
+        }
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+        warnings,
+    };
+}
+/**
+ * Format validation results for console output.
+ */
+export function formatValidationResult(result) {
+    const lines = [];
+    if (result.errors.length > 0) {
+        lines.push("❌ Errors:");
+        for (const e of result.errors) {
+            lines.push(`  [${e.taskId}] ${e.field}: ${e.message}`);
+        }
+    }
+    if (result.warnings.length > 0) {
+        lines.push("⚠️  Warnings:");
+        for (const w of result.warnings) {
+            lines.push(`  [${w.taskId}] ${w.field}: ${w.message}`);
+        }
+    }
+    if (result.valid && result.warnings.length === 0) {
+        lines.push("✅ All repo tasks pass validation");
+    }
+    return lines.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Snake_case detection (pre-parse helper)
+// ---------------------------------------------------------------------------
+/** Known snake_case → camelCase field mappings for common errors */
+const SNAKE_TO_CAMEL = {
+    feature_area: "featureArea",
+    canonical_docs: "canonicalDocs",
+    doc_coverage: "docCoverage",
+    reference_solution: "referenceSolution",
+};
+/**
+ * Detect snake_case field names in raw task YAML data.
+ *
+ * This runs BEFORE Zod parsing to provide a user-friendly error message
+ * when authors use framework-internal snake_case names instead of the
+ * camelCase names expected in repo task files.
+ *
+ * @param raw - Raw parsed YAML (before Zod validation)
+ * @param filename - Source filename for error messages
+ * @returns Array of warning messages (empty if no issues)
+ */
+export function detectSnakeCaseFields(raw, filename) {
+    const warnings = [];
+    if (!Array.isArray(raw))
+        return warnings;
+    for (let i = 0; i < raw.length; i++) {
+        const entry = raw[i];
+        if (typeof entry !== "object" || entry === null)
+            continue;
+        const obj = entry;
+        const taskId = typeof obj.id === "string" ? obj.id : `task[${i}]`;
+        for (const [snake, camel] of Object.entries(SNAKE_TO_CAMEL)) {
+            if (snake in obj) {
+                warnings.push(`[${filename}] ${taskId}: Found "${snake}" — repo tasks use camelCase. Did you mean "${camel}"?`);
+            }
+        }
+    }
+    return warnings;
+}

package/dist/adapters/api-client/api-client.d.ts

@@ -0,0 +1,75 @@
+/**
+ * api-client/api-client.ts — Typed HTTP client for the AILF API.
+ *
+ * Handles all communication with the AILF API: submission, polling,
+ * report fetching, and task validation. Uses Node's built-in `fetch`
+ * (Node 22+) — no external HTTP dependencies.
+ *
+ * The client is intentionally thin: it builds HTTP requests, parses
+ * responses into typed objects, and translates HTTP errors into typed
+ * error classes. It does NOT contain business logic.
+ *
+ * @see packages/api/src/routes/ — server-side route handlers
+ * @see docs/design-docs/cli-as-api-client.md — design doc
+ */
+import type { PipelineRequest } from "../../_vendor/ailf-core/index.d.ts";
+import type { ApiClientOptions, JobResponse, SubmitResponse, ValidationResponse, WaitOptions } from "./types.js";
+export declare class ApiClient {
+    private readonly apiKey;
+    private readonly baseUrl;
+    constructor(options: ApiClientOptions);
+    /**
+     * POST /v1/pipeline — submit an evaluation and return the job info.
+     *
+     * Returns HTTP 202 on success with a `SubmitResponse` containing the
+     * `jobId` and `statusUrl` for polling.
+     */
+    submitPipeline(request: PipelineRequest): Promise<SubmitResponse>;
+    /**
+     * GET /v1/jobs/:id — fetch job status.
+     *
+     * Supports long-polling via the `Prefer: wait=N` header. When
+     * `longPollSeconds` is set, the server holds the connection open
+     * for up to N seconds (max 25) waiting for a status change.
+     */
+    getJob(jobId: string, options?: {
+        longPollSeconds?: number;
+    }): Promise<JobResponse>;
+    /**
+     * Poll until the job reaches a terminal state (completed, failed,
+     * timed-out) or the timeout is exceeded.
+     *
+     * Uses long-polling for efficiency — the server holds the connection
+     * instead of the client sleeping between requests.
+     */
+    waitForCompletion(jobId: string, options?: WaitOptions): Promise<JobResponse>;
+    /**
+     * GET /v1/reports/:id/markdown — fetch the rendered markdown report.
+     *
+     * Returns the raw markdown string. Throws on 404 or error.
+     */
+    getReportMarkdown(reportId: string): Promise<string>;
+    /**
+     * POST /v1/validate/task — validate task definitions against the API.
+     *
+     * Returns a structured validation result. Does not throw on invalid
+     * tasks — the caller checks `response.valid`.
+     */
+    validateTasks(tasks: unknown[]): Promise<ValidationResponse>;
+    private get;
+    private post;
+    /**
+     * Build standard headers. Auth is always included.
+     */
+    private headers;
+    /**
+     * Fetch with retry on transient network errors.
+     * Retries up to MAX_RETRY_ATTEMPTS with exponential backoff.
+     * Only retries on network errors (TypeError from fetch), not HTTP errors.
+     */
+    private fetchWithRetry;
+    /**
+     * Parse a non-OK HTTP response and throw the appropriate typed error.
+     */
+    private handleErrorResponse;
+}

package/dist/adapters/api-client/api-client.js

@@ -0,0 +1,201 @@
+/**
+ * api-client/api-client.ts — Typed HTTP client for the AILF API.
+ *
+ * Handles all communication with the AILF API: submission, polling,
+ * report fetching, and task validation. Uses Node's built-in `fetch`
+ * (Node 22+) — no external HTTP dependencies.
+ *
+ * The client is intentionally thin: it builds HTTP requests, parses
+ * responses into typed objects, and translates HTTP errors into typed
+ * error classes. It does NOT contain business logic.
+ *
+ * @see packages/api/src/routes/ — server-side route handlers
+ * @see docs/design-docs/cli-as-api-client.md — design doc
+ */
+import { ApiAuthError, ApiConnectionError, ApiError, ApiTimeoutError, } from "./errors.js";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const DEFAULT_BASE_URL = "https://ailf-api.sanity.build";
+const MAX_LONG_POLL_SECONDS = 25;
+const DEFAULT_TIMEOUT_MS = 20 * 60 * 1000; // 20 minutes
+const MAX_RETRY_ATTEMPTS = 3;
+const RETRY_BASE_DELAY_MS = 2000;
+// ---------------------------------------------------------------------------
+// ApiClient
+// ---------------------------------------------------------------------------
+export class ApiClient {
+    apiKey;
+    baseUrl;
+    constructor(options) {
+        this.apiKey = options.apiKey;
+        this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+    }
+    // ─── Submission ─────────────────────────────────────────────────
+    /**
+     * POST /v1/pipeline — submit an evaluation and return the job info.
+     *
+     * Returns HTTP 202 on success with a `SubmitResponse` containing the
+     * `jobId` and `statusUrl` for polling.
+     */
+    async submitPipeline(request) {
+        return this.post("/v1/pipeline", request);
+    }
+    // ─── Job status ─────────────────────────────────────────────────
+    /**
+     * GET /v1/jobs/:id — fetch job status.
+     *
+     * Supports long-polling via the `Prefer: wait=N` header. When
+     * `longPollSeconds` is set, the server holds the connection open
+     * for up to N seconds (max 25) waiting for a status change.
+     */
+    async getJob(jobId, options) {
+        const headers = {};
+        if (options?.longPollSeconds) {
+            const wait = Math.min(options.longPollSeconds, MAX_LONG_POLL_SECONDS);
+            headers["Prefer"] = `wait=${wait}`;
+        }
+        return this.get(`/v1/jobs/${jobId}`, headers);
+    }
+    /**
+     * Poll until the job reaches a terminal state (completed, failed,
+     * timed-out) or the timeout is exceeded.
+     *
+     * Uses long-polling for efficiency — the server holds the connection
+     * instead of the client sleeping between requests.
+     */
+    async waitForCompletion(jobId, options) {
+        const { timeoutMs = DEFAULT_TIMEOUT_MS, onProgress, longPollSeconds = MAX_LONG_POLL_SECONDS, } = options ?? {};
+        const deadline = Date.now() + timeoutMs;
+        const terminalStatuses = new Set(["completed", "failed", "timed-out"]);
+        while (Date.now() < deadline) {
+            let job;
+            try {
+                job = await this.getJob(jobId, { longPollSeconds });
+            }
+            catch (err) {
+                // Retry on transient errors during polling
+                if (err instanceof ApiConnectionError && Date.now() < deadline) {
+                    await sleep(RETRY_BASE_DELAY_MS);
+                    continue;
+                }
+                throw err;
+            }
+            onProgress?.(job);
+            if (terminalStatuses.has(job.status)) {
+                return job;
+            }
+            // Long-polling already handles the delay; only add a small
+            // backoff if the server responded instantly (non-terminal).
+        }
+        throw new ApiTimeoutError(jobId, timeoutMs);
+    }
+    // ─── Reports ────────────────────────────────────────────────────
+    /**
+     * GET /v1/reports/:id/markdown — fetch the rendered markdown report.
+     *
+     * Returns the raw markdown string. Throws on 404 or error.
+     */
+    async getReportMarkdown(reportId) {
+        const url = `${this.baseUrl}/v1/reports/${reportId}/markdown`;
+        const response = await this.fetchWithRetry(url, {
+            method: "GET",
+            headers: this.headers({ Accept: "text/markdown" }),
+        });
+        if (!response.ok) {
+            await this.handleErrorResponse(response, url);
+        }
+        return response.text();
+    }
+    // ─── Validation ─────────────────────────────────────────────────
+    /**
+     * POST /v1/validate/task — validate task definitions against the API.
+     *
+     * Returns a structured validation result. Does not throw on invalid
+     * tasks — the caller checks `response.valid`.
+     */
+    async validateTasks(tasks) {
+        return this.post("/v1/validate/task", { tasks });
+    }
+    // ─── Internal HTTP methods ──────────────────────────────────────
+    async get(path, extraHeaders) {
+        const url = `${this.baseUrl}${path}`;
+        const response = await this.fetchWithRetry(url, {
+            method: "GET",
+            headers: this.headers(extraHeaders),
+        });
+        if (!response.ok) {
+            await this.handleErrorResponse(response, url);
+        }
+        return response.json();
+    }
+    async post(path, body) {
+        const url = `${this.baseUrl}${path}`;
+        const response = await this.fetchWithRetry(url, {
+            method: "POST",
+            headers: this.headers({ "Content-Type": "application/json" }),
+            body: JSON.stringify(body),
+        });
+        // 202 is success for async operations
+        if (!response.ok && response.status !== 202) {
+            await this.handleErrorResponse(response, url);
+        }
+        return response.json();
+    }
+    /**
+     * Build standard headers. Auth is always included.
+     */
+    headers(extra) {
+        return {
+            Authorization: `Bearer ${this.apiKey}`,
+            ...extra,
+        };
+    }
+    /**
+     * Fetch with retry on transient network errors.
+     * Retries up to MAX_RETRY_ATTEMPTS with exponential backoff.
+     * Only retries on network errors (TypeError from fetch), not HTTP errors.
+     */
+    async fetchWithRetry(url, init) {
+        let lastError;
+        for (let attempt = 0; attempt < MAX_RETRY_ATTEMPTS; attempt++) {
+            try {
+                return await fetch(url, init);
+            }
+            catch (err) {
+                lastError = err;
+                // Only retry on network-level errors (not HTTP status errors)
+                if (attempt < MAX_RETRY_ATTEMPTS - 1) {
+                    const delay = RETRY_BASE_DELAY_MS * 2 ** attempt;
+                    await sleep(delay);
+                }
+            }
+        }
+        throw new ApiConnectionError(`Could not reach the AILF API at ${url}`, url, lastError);
+    }
+    /**
+     * Parse a non-OK HTTP response and throw the appropriate typed error.
+     */
+    async handleErrorResponse(response, url) {
+        let body;
+        try {
+            body = (await response.json());
+        }
+        catch {
+            // Non-JSON error response (e.g., HTML from CDN)
+        }
+        const code = body?.error?.code ?? `http_${response.status}`;
+        const message = body?.error?.message ??
+            `HTTP ${response.status} ${response.statusText} (${url})`;
+        if (response.status === 401 || response.status === 403) {
+            throw new ApiAuthError(message, code, response.status);
+        }
+        throw new ApiError(message, code, response.status, body?.error?.param);
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}