create-windy 0.8.2 → 0.9.1

package/template/apps/agent-server/runtime/evaluation-plugin.ts

@@ -0,0 +1,115 @@
+import { randomUUID } from "node:crypto";
+import { z } from "zod";
+import { json, readJson, SafeError } from "./contracts.js";
+import { canonicalDigest } from "./context-plugin.js";
+import { terminal, type RuntimeStore } from "./store.js";
+const sha = z.string().regex(/^sha256:[a-f0-9]{64}$/);
+const evaluationSchema = z
+  .object({
+    runIds: z
+      .array(z.string().trim().min(1).max(160))
+      .min(1)
+      .max(100)
+      .refine((ids) => new Set(ids).size === ids.length),
+    evaluationSetKey: z.string().min(1).max(160),
+    evaluationSetVersion: z.string().min(1).max(80),
+    runnerVersion: z.string().min(1).max(80),
+    runtimePin: z
+      .object({
+        key: z.string().min(1).max(160),
+        identity: z.string().min(1).max(200),
+        releaseDigest: sha,
+        offersDigest: sha,
+      })
+      .strict(),
+    resultDigest: sha,
+    metrics: z
+      .record(z.string().min(1).max(160), z.number().finite())
+      .refine(
+        (value) =>
+          Object.keys(value).length >= 1 && Object.keys(value).length <= 100,
+      ),
+  })
+  .strict();
+/** Stores externally measured evidence against actual terminal DSH Runs. */
+export class EvaluationPlugin {
+  constructor(private store: RuntimeStore) {
+    store.db.exec(`CREATE TABLE IF NOT EXISTS evaluations (
+      id TEXT PRIMARY KEY, release_id TEXT NOT NULL, dataset_key TEXT NOT NULL,
+      dataset_version TEXT NOT NULL, evidence_digest TEXT NOT NULL,
+      evidence TEXT NOT NULL, created_at TEXT NOT NULL,
+      UNIQUE(release_id,dataset_key,dataset_version));`);
+    store.db
+      .prepare("DELETE FROM evaluations WHERE created_at<?")
+      .run(new Date(Date.now() - 90 * 86400000).toISOString());
+  }
+  async handle(request: Request): Promise<Response> {
+    const body = evaluationSchema.parse(await readJson(request, 262144));
+    request.signal.throwIfAborted();
+    const runs = body.runIds.map((id) => this.store.get(id));
+    const releaseIds = new Set(
+        runs.map((run) => String(run.request.release.releaseId ?? "")),
+      ),
+      operations = new Set(runs.map((run) => run.request.operationKey));
+    if (
+      runs.some((run) => !terminal(run.status)) ||
+      releaseIds.size !== 1 ||
+      operations.size !== 1 ||
+      new Set(runs.map((run) => canonicalDigest(run.request.release))).size !==
+        1
+    )
+      throw new SafeError(
+        "RUN_CONFLICT",
+        "评估必须绑定同一 Operation Release 的已结束 Run",
+      );
+    const releaseId = [...releaseIds][0]!;
+    if (!releaseId)
+      throw new SafeError(
+        "RUN_CONFLICT",
+        "评估绑定的 Release 不存在",
+        false,
+        404,
+      );
+    const evidence = {
+      schemaVersion: 1,
+      runIds: body.runIds,
+      operationKey: [...operations][0],
+      runnerVersion: body.runnerVersion,
+      runtimePin: body.runtimePin,
+      resultDigest: body.resultDigest,
+      metrics: body.metrics,
+    };
+    const evidenceDigest = canonicalDigest(evidence);
+    const existing = this.store.db
+      .prepare(
+        "SELECT id,evidence_digest FROM evaluations WHERE release_id=? AND dataset_key=? AND dataset_version=?",
+      )
+      .get(releaseId, body.evaluationSetKey, body.evaluationSetVersion) as
+      | { id: string; evidence_digest: string }
+      | undefined;
+    if (existing) {
+      if (existing.evidence_digest !== evidenceDigest)
+        throw new SafeError("RUN_CONFLICT", "评估版本已经绑定其他结果");
+      return json({ evaluationId: existing.id, releaseId });
+    }
+    const count = this.store.db
+      .prepare("SELECT COUNT(*) AS count FROM evaluations")
+      .get() as { count: number };
+    if (count.count >= 10000)
+      throw new SafeError("BUDGET_EXCEEDED", "评估留存容量已满", true, 429);
+    request.signal.throwIfAborted();
+    const evaluationId = `eval_${randomUUID().replaceAll("-", "")}`;
+    this.store.db
+      .prepare("INSERT INTO evaluations VALUES(?,?,?,?,?,?,?)")
+      .run(
+        evaluationId,
+        releaseId,
+        body.evaluationSetKey,
+        body.evaluationSetVersion,
+        evidenceDigest,
+        JSON.stringify(evidence),
+        new Date().toISOString(),
+      );
+    return json({ evaluationId, releaseId });
+  }
+}

package/template/apps/agent-server/runtime/evaluation.nodecheck.ts

@@ -0,0 +1,194 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { runtime, request, input } from "./runtime-test-fixtures.js";
+import { validateStructuredOutput } from "./output-validation.js";
+function evidence(runIds: string[]) {
+  return {
+    runIds,
+    evaluationSetKey: "analysis.eval.copilot-general",
+    evaluationSetVersion: "1",
+    runnerVersion: "copilot-eval-test",
+    runtimePin: {
+      key: "test-runtime",
+      identity: "test-deployment",
+      releaseDigest: `sha256:${"a".repeat(64)}`,
+      offersDigest: `sha256:${"b".repeat(64)}`,
+    },
+    resultDigest: `sha256:${"c".repeat(64)}`,
+    metrics: { passed: 1, scenarioCount: runIds.length, assertionRate: 1 },
+  };
+}
+async function completed(
+  handler: ReturnType<typeof runtime>,
+  key: string,
+  operationKey = "test.operation",
+) {
+  const body = input();
+  body.operationKey = operationKey;
+  const start = await handler(request("/api/agent/runs", "POST", body, key));
+  assert.equal(start?.status, 200);
+  const run = (await start!.json()) as { runId: string; eventsUrl: string };
+  const events = await handler(request(run.eventsUrl));
+  assert.match(await events!.text(), /event: run.completed/);
+  return run;
+}
+test("evaluation records actual DSH runs and persists idempotent external metrics across restart", async () => {
+  const directory = mkdtempSync(join(tmpdir(), "dsh-evaluation-")),
+    database = join(directory, "runtime.sqlite");
+  let handler = runtime({ AGENT_RUNTIME_DATABASE_PATH: database });
+  try {
+    const run = await completed(handler, "evaluation-run"),
+      body = evidence([run.runId]);
+    const response = await handler(
+      request("/api/agent/evaluations", "POST", body),
+    );
+    assert.equal(response?.status, 200);
+    const recorded = await response!.json();
+    assert.match((recorded as { evaluationId: string }).evaluationId, /^eval_/);
+    assert.equal((recorded as { releaseId: string }).releaseId, "test-release");
+    await handler.close();
+    handler = runtime({ AGENT_RUNTIME_DATABASE_PATH: database });
+    const replay = await handler(
+      request("/api/agent/evaluations", "POST", body),
+    );
+    assert.deepEqual(await replay!.json(), recorded);
+    const conflict = await handler(
+      request("/api/agent/evaluations", "POST", {
+        ...body,
+        metrics: { passed: 0 },
+      }),
+    );
+    assert.equal(conflict?.status, 409);
+  } finally {
+    await handler.close();
+    rmSync(directory, { recursive: true });
+  }
+});
+test("evaluation enforces auth, terminal state, binding consistency and bounded request shape", async () => {
+  const handler = runtime();
+  try {
+    const body = input();
+    body.userPrompt = "hold-model";
+    const start = await handler(
+      request("/api/agent/runs", "POST", body, "pending-evaluation"),
+    );
+    const pending = (await start!.json()) as {
+      runId: string;
+      eventsUrl: string;
+    };
+    const inProgress = await handler(
+      request("/api/agent/evaluations", "POST", evidence([pending.runId])),
+    );
+    assert.equal(inProgress?.status, 409);
+    await handler(
+      request(`/api/agent/runs/${pending.runId}/cancel`, "POST", {
+        reason: "evaluation cancelled",
+      }),
+    );
+    const events = await handler(request(pending.eventsUrl));
+    assert.match(await events!.text(), /event: run.cancelled/);
+    const cancelled = await handler(
+      request("/api/agent/evaluations", "POST", {
+        ...evidence([pending.runId]),
+        metrics: { passed: 0, cancelled: 1 },
+      }),
+    );
+    assert.equal(cancelled?.status, 200);
+    const forbidden = await handler(
+      new Request("http://runtime/api/agent/evaluations", {
+        method: "POST",
+        body: JSON.stringify(evidence([pending.runId])),
+      }),
+    );
+    assert.equal(forbidden?.status, 403);
+    const unknown = await handler(
+      request("/api/agent/evaluations", "POST", evidence(["run_missing"])),
+    );
+    assert.equal(unknown?.status, 404);
+    const excessive = await handler(
+      request(
+        "/api/agent/evaluations",
+        "POST",
+        evidence(Array.from({ length: 101 }, (_, index) => `run_${index}`)),
+      ),
+    );
+    assert.equal(excessive?.status, 400);
+    const duplicate = await handler(
+      request(
+        "/api/agent/evaluations",
+        "POST",
+        evidence([pending.runId, pending.runId]),
+      ),
+    );
+    assert.equal(duplicate?.status, 400);
+    const other = await completed(
+      handler,
+      "other-operation",
+      "test.other-operation",
+    );
+    const mixed = await handler(
+      request(
+        "/api/agent/evaluations",
+        "POST",
+        evidence([pending.runId, other.runId]),
+      ),
+    );
+    assert.equal(mixed?.status, 409);
+  } finally {
+    await handler.close();
+  }
+});
+test("structured-output diagnostics preserve safe schema kind and path without provider content", () => {
+  const logs: string[] = [],
+    context = {
+      runId: "run_test",
+      operationKey: "analysis.copilot.global",
+      outputSchemaKey: "copilot.output.v1",
+    };
+  assert.throws(
+    () =>
+      validateStructuredOutput(
+        "private unstructured model content",
+        { type: "object" },
+        context,
+        (value) => logs.push(value),
+      ),
+    /格式校验/,
+  );
+  assert.equal(JSON.parse(logs[0]!).validationKind, "json-syntax");
+  assert.throws(
+    () =>
+      validateStructuredOutput(
+        '{"answer":123}',
+        { type: "object", properties: { answer: { type: "string" } } },
+        context,
+        (value) => logs.push(value),
+      ),
+    /格式校验/,
+  );
+  assert.deepEqual(
+    {
+      kind: JSON.parse(logs[1]!).validationKind,
+      path: JSON.parse(logs[1]!).instancePath,
+    },
+    { kind: "schema-instance", path: "/answer" },
+  );
+  assert.throws(
+    () =>
+      validateStructuredOutput(
+        "{}",
+        { type: "not-a-schema-type" },
+        context,
+        (value) => logs.push(value),
+      ),
+    /格式校验/,
+  );
+  assert.equal(JSON.parse(logs[2]!).validationKind, "schema-definition");
+  assert.doesNotMatch(
+    logs.join("\n"),
+    /private unstructured model content|"answer":123/,
+  );
+});

package/template/apps/agent-server/runtime/events.ts

@@ -0,0 +1,71 @@
+import { SafeError } from "./contracts.js";
+import { terminal, type RuntimeStore } from "./store.js";
+export function publicEvent(event: Record<string, unknown>) {
+  const {
+    attestation: _attestation,
+    sourcePreview: _preview,
+    sourceSnapshotDigest: _digest,
+    ...value
+  } = event;
+  return value;
+}
+export function eventResponse(
+  request: Request,
+  id: string,
+  store: RuntimeStore,
+) {
+  const url = new URL(request.url),
+    query = url.searchParams.get("after") ?? "0",
+    header = request.headers.get("Last-Event-ID") ?? "0";
+  if (!/^\d+$/.test(query) || !/^\d+$/.test(header))
+    throw new SafeError("INVALID_INPUT", "事件游标无效", false, 400);
+  let after = Math.max(Number(query), Number(header));
+  if (!Number.isSafeInteger(after))
+    throw new SafeError("INVALID_INPUT", "事件游标无效", false, 400);
+  const encoder = new TextEncoder();
+  let cleanup: (() => void) | undefined;
+  let cancelled = false;
+  const body = new ReadableStream<Uint8Array>({
+    start(sink) {
+      let done = false;
+      const close = () => {
+        if (done) return;
+        done = true;
+        unsubscribe();
+        clearInterval(heartbeat);
+        request.signal.removeEventListener("abort", close);
+        if (!cancelled) sink.close();
+      };
+      const flush = () => {
+        if (done) return;
+        for (const event of store.events(id, after)) {
+          after = event.sequence;
+          sink.enqueue(
+            encoder.encode(
+              `id: ${event.sequence}\nevent: ${event.type}\ndata: ${JSON.stringify(publicEvent(event))}\n\n`,
+            ),
+          );
+        }
+        if (terminal(store.get(id).status)) close();
+      };
+      const unsubscribe = store.subscribe(id, flush),
+        heartbeat = setInterval(() => {
+          if (!done) sink.enqueue(encoder.encode(": heartbeat\n\n"));
+        }, 15000);
+      cleanup = close;
+      request.signal.addEventListener("abort", close, { once: true });
+      flush();
+    },
+    cancel() {
+      cancelled = true;
+      cleanup?.();
+    },
+  });
+  return new Response(body, {
+    headers: {
+      "content-type": "text/event-stream",
+      "cache-control": "no-cache, no-transform",
+      "x-accel-buffering": "no",
+    },
+  });
+}

package/template/apps/agent-server/runtime/governance.ts

@@ -0,0 +1,194 @@
+import { SafeError, type ProviderPolicy } from "./contracts.js";
+import type { RuntimeStore } from "./store.js";
+const detectors = [
+  /\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi,
+  /\b(?:api[_-]?key|secret|password|token)\s*[:=]\s*\S+/gi,
+  /\b1[3-9]\d{9}\b/g,
+  /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
+  /\b\d{17}[\dXx]\b/g,
+  /\b(?:\d[ -]?){16,30}\b/g,
+];
+export function governedValue(
+  value: unknown,
+  action: "redact" | "reject",
+): unknown {
+  if (typeof value === "string") {
+    let result = value;
+    for (const pattern of detectors) {
+      pattern.lastIndex = 0;
+      if (pattern.test(result)) {
+        if (action === "reject")
+          throw new SafeError(
+            "FORBIDDEN",
+            "Provider 请求被字段治理策略拒绝",
+            false,
+            403,
+          );
+        pattern.lastIndex = 0;
+        result = result.replace(pattern, "[REDACTED]");
+      }
+    }
+    return result;
+  }
+  if (Array.isArray(value))
+    return value.map((item) => governedValue(item, action));
+  if (value && typeof value === "object")
+    return Object.fromEntries(
+      Object.entries(value).map(([key, item]) => [
+        key,
+        governedValue(item, action),
+      ]),
+    );
+  return value;
+}
+export function governProvider(
+  messages: Record<string, unknown>[],
+  tools: Record<string, unknown>[],
+  outputSchema: Record<string, unknown> | null | undefined,
+  policy: ProviderPolicy,
+  audit: RuntimeStore,
+  metadata: Record<string, unknown>,
+) {
+  const rules = new Map(
+    policy.envelopeFields.map((rule) => [rule.field, rule]),
+  );
+  let redacted = false;
+  const classified = new Set<string>();
+  function apply(field: string, value: unknown): unknown {
+    const rule = rules.get(field);
+    if (!rule)
+      throw new SafeError(
+        "FORBIDDEN",
+        `Provider 未授权发送字段：${field}`,
+        false,
+        403,
+      );
+    classified.add(rule.classification);
+    const governed = governedValue(value, rule.dlpAction);
+    redacted ||= JSON.stringify(governed) !== JSON.stringify(value);
+    return governed;
+  }
+  try {
+    if (rules.size !== policy.envelopeFields.length)
+      throw new SafeError("FORBIDDEN", "Provider 字段规则重复", false, 403);
+    const roles: Record<string, string> = {
+      system: "instructions",
+      developer: "instructions",
+      user: "user-prompt",
+      assistant: "provider-output",
+      tool: "tool-results",
+    };
+    const governedMessages = messages.map((message) => {
+      const field = roles[String(message.role)];
+      if (!field)
+        throw new SafeError("FORBIDDEN", "Provider 消息角色无效", false, 403);
+      if (Array.isArray(message.content)) {
+        const content = message.content.map((raw: unknown) => {
+          const part = raw as Record<string, unknown>;
+          if (part.type === "text") return apply(field, part);
+          const image = part.image_url as Record<string, unknown> | undefined;
+          if (
+            part.type !== "image_url" ||
+            typeof image?.url !== "string" ||
+            !/^data:image\/(png|jpeg|webp);base64,[A-Za-z0-9+/=]+$/.test(
+              image.url,
+            )
+          )
+            throw new SafeError("FORBIDDEN", "图片输入结构无效", false, 403);
+          const rule = rules.get("artifact-content");
+          if (!rule)
+            throw new SafeError(
+              "FORBIDDEN",
+              "Provider 未授权发送图片",
+              false,
+              403,
+            );
+          classified.add(rule.classification);
+          return part;
+        });
+        const { content: _content, ...metadata } = message;
+        return {
+          ...(apply(field, metadata) as Record<string, unknown>),
+          content,
+        };
+      }
+      return apply(field, message) as Record<string, unknown>;
+    });
+    const governedTools = tools.length
+      ? (apply("tool-definitions", tools) as Record<string, unknown>[])
+      : [];
+    const schema = outputSchema
+      ? (apply("output-schema", outputSchema) as Record<string, unknown>)
+      : undefined;
+    const bytes = Buffer.byteLength(
+      JSON.stringify({
+        messages: governedMessages,
+        tools: governedTools,
+        schema,
+      }),
+    );
+    audit.audit({
+      ...metadata,
+      classified: [...classified].sort(),
+      sentBytes: bytes,
+      decision: redacted ? "redact" : "allow",
+    });
+    return { messages: governedMessages, tools: governedTools, schema, bytes };
+  } catch (error) {
+    audit.audit({
+      ...metadata,
+      classified: [...classified].sort(),
+      sentBytes: 0,
+      decision: "reject",
+    });
+    throw error;
+  }
+}
+export async function* sseJson(
+  response: Response,
+  byteLimit: number,
+  signal?: AbortSignal,
+): AsyncGenerator<Record<string, unknown>> {
+  if (!response.body)
+    throw new SafeError("PROVIDER_UNAVAILABLE", "模型响应为空", true, 503);
+  const reader = response.body.getReader();
+  const abort = () => {
+    void reader.cancel().catch(() => {});
+  };
+  signal?.addEventListener("abort", abort, { once: true });
+  if (signal?.aborted) abort();
+  const decoder = new TextDecoder();
+  let buffer = "",
+    bytes = 0;
+  try {
+    for (;;) {
+      const next = await reader.read();
+      if (next.done) break;
+      bytes += next.value.byteLength;
+      if (bytes > byteLimit)
+        throw new SafeError("BUDGET_EXCEEDED", "模型响应超过上限");
+      buffer += decoder.decode(next.value, { stream: true });
+      let lineEnd: number;
+      while ((lineEnd = buffer.indexOf("\n")) >= 0) {
+        const line = buffer.slice(0, lineEnd).trim();
+        buffer = buffer.slice(lineEnd + 1);
+        if (!line.startsWith("data:")) continue;
+        const value = line.slice(5).trim();
+        if (value === "[DONE]") return;
+        if (value) {
+          const parsed: unknown = JSON.parse(value);
+          if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+            throw new SafeError(
+              "PROVIDER_UNAVAILABLE",
+              "模型返回协议无效",
+              true,
+            );
+          yield parsed as Record<string, unknown>;
+        }
+      }
+    }
+  } finally {
+    signal?.removeEventListener("abort", abort);
+    await reader.cancel();
+  }
+}