create-windy 0.5.2 → 0.6.0

package/template/apps/server/src/agent/screenshot-intake-v2.ts

@@ -0,0 +1,350 @@
+import type { ModuleManifest } from "@southwind-ai/modules";
+import type {
+  AiRuntimeDeploymentV1,
+  ModuleAiArtifactRef,
+  ModuleAiScreenshotIntakeCreateV2,
+  ModuleAiScreenshotPlanV2,
+} from "@southwind-ai/server-sdk";
+import type { RequestGuardContext } from "../guards.js";
+import { authorizeScreenshotOperation } from "./screenshot-intake-authorization.js";
+import { pngPixels } from "./screenshot-png.js";
+import { screenshotIntakeUnavailable as unavailable } from "./screenshot-intake-errors.js";
+import {
+  durableScreenshotProducer,
+  type ProjectAiScreenshotIntakeFactoryV2,
+} from "./screenshot-intake-durable.js";
+import {
+  callBoundedScreenshotProducer,
+  cancelScreenshotProducer,
+  rejectDuplicateScreenshotProducer,
+} from "./screenshot-intake-producer.js";
+import type {
+  ScreenshotArtifactValidator,
+  ScreenshotIntakeRecordV2,
+} from "./screenshot-intake-record.js";
+import { restoreScreenshotRecord } from "./screenshot-intake-recovery.js";
+import { reserveProducerScreenshot } from "./screenshot-intake-reservation.js";
+import type { ScreenshotIntakeServerRuntime } from "./screenshot-intake-runtime-types.js";
+import { sweepScreenshotRecords } from "./screenshot-intake-sweep.js";
+import {
+  assertScreenshotDeclaration,
+  bindingForScreenshot,
+  assertScreenshotOperation,
+  assertScreenshotProducerPlan,
+  sameScreenshotArtifact,
+  sameScreenshotBinding,
+  sameScreenshotDeployment,
+  screenshotModuleName,
+  screenshotPurpose,
+} from "./screenshot-intake-v2-validation.js";
+
+const MAX_ENTRIES = 1_024;
+const MAX_TTL_MS = 60_000;
+const SWEEP_INTERVAL_MS = 10_000;
+
+export type { ProjectAiScreenshotIntakeFactoryV2 } from "./screenshot-intake-durable.js";
+
+export class ScreenshotIntakeRuntimeV2 {
+  readonly #records = new Map<string, ScreenshotIntakeRecordV2>();
+  readonly #sweeper: ReturnType<typeof setInterval>;
+
+  constructor(
+    private readonly modules: readonly ModuleManifest[],
+    private readonly runtime: ScreenshotIntakeServerRuntime,
+    private readonly factory: ProjectAiScreenshotIntakeFactoryV2 | undefined,
+    private readonly validateArtifact: ScreenshotArtifactValidator,
+    private readonly now: () => Date = () => new Date(),
+    private readonly cleanupPersistent:
+      | (() => Promise<void>)
+      | undefined = undefined,
+  ) {
+    this.#sweeper = setInterval(() => void this.sweep(), SWEEP_INTERVAL_MS);
+    (this.#sweeper as { unref?: () => void }).unref?.();
+  }
+
+  async dispose(): Promise<void> {
+    clearInterval(this.#sweeper);
+    await Promise.allSettled(
+      [...this.#records.values()]
+        .filter((record) => !record.durable)
+        .map((record) => this.cancel(record)),
+    );
+    this.#records.clear();
+  }
+
+  async create(
+    input: ModuleAiScreenshotIntakeCreateV2,
+    context: RequestGuardContext,
+    signal: AbortSignal,
+  ): Promise<ModuleAiScreenshotPlanV2> {
+    await this.sweep();
+    if (!this.factory || this.#records.size >= MAX_ENTRIES) throw unavailable();
+    assertScreenshotDeclaration(input);
+    const producer = this.factory(context, signal);
+    if (!producer || signal.aborted) throw unavailable();
+    let plan:
+      | (ModuleAiScreenshotPlanV2 & {
+          readonly operationKey: string;
+        })
+      | undefined;
+    let deployment: AiRuntimeDeploymentV1;
+    try {
+      plan = await callBoundedScreenshotProducer(
+        () => producer.create(input),
+        unavailable,
+        (late) =>
+          cancelScreenshotProducer(producer, late.intakeToken, unavailable),
+      );
+      assertScreenshotProducerPlan(plan, this.now());
+      const entry = await authorizeScreenshotOperation(
+        this.modules,
+        this.runtime,
+        plan.operationKey,
+        context,
+      );
+      assertScreenshotOperation(entry.operation, input);
+      if (!entry.operation.readiness) throw unavailable();
+      deployment = entry.operation.readiness.expectedDeployment;
+    } catch {
+      if (plan)
+        await cancelScreenshotProducer(producer, plan.intakeToken, unavailable);
+      throw unavailable();
+    }
+    if (!plan) throw unavailable();
+    const expiresAtMs = Math.min(
+      Date.parse(plan.expiresAt),
+      this.now().getTime() + MAX_TTL_MS,
+    );
+    const expiresAt = new Date(expiresAtMs).toISOString();
+    const token = plan.intakeToken;
+    if (this.#records.has(token))
+      await rejectDuplicateScreenshotProducer(producer, token, unavailable);
+    this.#records.set(token, {
+      ...bindingForScreenshot(context),
+      token,
+      producer,
+      producerToken: plan.intakeToken,
+      durable: durableScreenshotProducer(producer),
+      moduleName: screenshotModuleName(this.modules, plan.operationKey),
+      operationKey: plan.operationKey,
+      purpose: screenshotPurpose(plan.operationKey),
+      pageId: input.pageId,
+      contextDigest: input.contextDigest,
+      deployment,
+      expiresAtMs,
+      mediaType: input.mediaType,
+      byteSize: input.byteSize,
+      pixels: input.pixels,
+      state: "created",
+    });
+    return Object.freeze({
+      contractVersion: 2,
+      intakeToken: token,
+      expiresAt,
+    });
+  }
+
+  async upload(
+    token: string,
+    mediaType: string,
+    bytes: Uint8Array,
+    context: RequestGuardContext,
+    signal: AbortSignal = AbortSignal.timeout(10_000),
+  ): Promise<void> {
+    const record = await this.claimable(token, context, signal);
+    await authorizeScreenshotOperation(
+      this.modules,
+      this.runtime,
+      record.operationKey,
+      context,
+    );
+    if (
+      bytes.byteLength !== record.byteSize ||
+      mediaType !== record.mediaType
+    ) {
+      if (record.state === "created") await this.cancel(record);
+      throw unavailable();
+    }
+    if (mediaType !== "image/png" || pngPixels(bytes) !== record.pixels) {
+      if (record.state === "created") await this.cancel(record);
+      throw unavailable();
+    }
+    if (record.state !== "created") {
+      if (
+        record.durable &&
+        record.artifact &&
+        ["uploaded", "reserving", "bound"].includes(record.state) &&
+        sameScreenshotArtifact(record.artifact, record, bytes, this.now())
+      )
+        return;
+      throw unavailable();
+    }
+    record.state = "uploading";
+    try {
+      const artifact = await callBoundedScreenshotProducer(
+        () =>
+          record.producer.upload({
+            intakeToken: record.producerToken,
+            bytes,
+            mediaType,
+          }),
+        unavailable,
+        () => this.cancel(record),
+      );
+      const validated = this.validateArtifact(
+        artifact,
+        record.actorId,
+        record.moduleName,
+        record.purpose,
+      );
+      if (!sameScreenshotArtifact(validated, record, bytes, this.now()))
+        throw unavailable();
+      record.artifact = Object.freeze({ ...validated });
+      record.state = "uploaded";
+    } catch {
+      await this.cancel(record);
+      throw unavailable();
+    }
+  }
+
+  async reserve(
+    token: string,
+    operationKey: string,
+    deployment: AiRuntimeDeploymentV1 | undefined,
+    context: RequestGuardContext,
+    signal: AbortSignal = AbortSignal.timeout(10_000),
+  ): Promise<{
+    artifact: ModuleAiArtifactRef;
+    bind(runId: string): Promise<void>;
+    cancel(): Promise<void>;
+  }> {
+    const record = await this.claimable(token, context, signal);
+    await authorizeScreenshotOperation(
+      this.modules,
+      this.runtime,
+      record.operationKey,
+      context,
+    );
+    if (
+      !record.artifact ||
+      (!record.durable && record.state !== "uploaded") ||
+      (record.durable &&
+        !["uploaded", "reserving", "bound"].includes(record.state)) ||
+      record.operationKey !== operationKey ||
+      !deployment ||
+      !sameScreenshotDeployment(record.deployment, deployment)
+    ) {
+      await this.cancel(record);
+      throw unavailable();
+    }
+    await reserveProducerScreenshot(
+      record.producer,
+      record.producerToken,
+      unavailable,
+      () => this.cancel(record),
+    );
+    if (record.state === "uploaded") record.state = "reserving";
+    return Object.freeze({
+      artifact: record.artifact,
+      bind: async (runId: string) => {
+        if (
+          !runId.trim() ||
+          (record.state !== "reserving" && record.state !== "bound")
+        )
+          throw unavailable();
+        record.state = "binding";
+        try {
+          await callBoundedScreenshotProducer(
+            () =>
+              record.producer.bind({
+                intakeToken: record.producerToken,
+                runId,
+              }),
+            unavailable,
+            () => this.cancel(record),
+          );
+        } catch {
+          await this.cancel(record);
+          throw unavailable();
+        }
+        record.state = "bound";
+        this.#records.delete(record.token);
+      },
+      cancel: () => this.cancel(record),
+    });
+  }
+
+  async cancelToken(
+    token: string,
+    context: RequestGuardContext,
+    signal: AbortSignal = AbortSignal.timeout(10_000),
+  ): Promise<void> {
+    const record = await this.claimable(token, context, signal);
+    await authorizeScreenshotOperation(
+      this.modules,
+      this.runtime,
+      record.operationKey,
+      context,
+    );
+    if (record.state === "binding" || record.state === "bound")
+      throw unavailable();
+    await this.cancel(record);
+  }
+
+  async sweep(): Promise<void> {
+    await this.cleanupPersistent?.().catch(() => undefined);
+    await sweepScreenshotRecords(
+      this.#records,
+      this.now().getTime(),
+      (record) => this.cancel(record),
+    );
+  }
+
+  private async claimable(
+    token: string,
+    context: RequestGuardContext,
+    signal: AbortSignal,
+  ): Promise<ScreenshotIntakeRecordV2> {
+    let record = this.#records.get(token);
+    if (!record) {
+      const producer = this.factory?.(context, signal);
+      if (!producer || !durableScreenshotProducer(producer))
+        throw unavailable();
+      const snapshot = await callBoundedScreenshotProducer(
+        () => producer.recover(token),
+        unavailable,
+      );
+      if (!snapshot) throw unavailable();
+      const recovered = restoreScreenshotRecord({
+        token,
+        producer,
+        snapshot,
+        modules: this.modules,
+        context,
+        validateArtifact: this.validateArtifact,
+        now: this.now(),
+      });
+      record = this.#records.get(token) ?? recovered;
+      this.#records.set(token, record);
+    }
+    if (record.expiresAtMs <= this.now().getTime()) {
+      void this.cancel(record);
+      throw unavailable();
+    }
+    if (!sameScreenshotBinding(record, bindingForScreenshot(context)))
+      throw unavailable();
+    return record;
+  }
+
+  private async cancel(record: ScreenshotIntakeRecordV2): Promise<void> {
+    if (record.cleanup) return await record.cleanup;
+    record.state = "cleanup";
+    record.cleanup = callBoundedScreenshotProducer(
+      () => record.producer.cancel({ intakeToken: record.producerToken }),
+      unavailable,
+    )
+      .catch(() => undefined)
+      .finally(() => this.#records.delete(record.token));
+    await record.cleanup;
+  }
+}

package/template/apps/server/src/agent/screenshot-png.ts

@@ -0,0 +1,15 @@
+/** Returns decoded IHDR dimensions only for a structurally valid PNG header. */
+export function pngPixels(bytes: Uint8Array): number | undefined {
+  const signature = [137, 80, 78, 71, 13, 10, 26, 10];
+  if (
+    bytes.byteLength < 24 ||
+    signature.some((value, index) => bytes[index] !== value) ||
+    String.fromCharCode(...bytes.slice(12, 16)) !== "IHDR"
+  )
+    return undefined;
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const width = view.getUint32(16);
+  const height = view.getUint32(20);
+  const pixels = width * height;
+  return Number.isSafeInteger(pixels) && pixels > 0 ? pixels : undefined;
+}

package/template/apps/server/src/agent/screenshot-run-binding.ts

@@ -0,0 +1,34 @@
+import type {
+  AiRuntimeDeploymentV1,
+  ModuleAiArtifactRef,
+} from "@southwind-ai/server-sdk";
+import type { RequestGuardContext } from "../guards.js";
+import { AgentRunFacadeError } from "./run-errors.js";
+import type { ScreenshotIntakeRuntimeV2 } from "./screenshot-intake-v2.js";
+
+export interface ScreenshotRunBinding {
+  readonly artifact: ModuleAiArtifactRef;
+  bind(runId: string): Promise<void>;
+  cancel(): Promise<void>;
+}
+
+export async function reserveScreenshotForRun(
+  intake: ScreenshotIntakeRuntimeV2 | undefined,
+  token: string | undefined,
+  operationKey: string,
+  deployment: AiRuntimeDeploymentV1 | undefined,
+  context: RequestGuardContext,
+  signal: AbortSignal,
+): Promise<ScreenshotRunBinding | undefined> {
+  if (!token) return undefined;
+  if (!intake) throw unavailable();
+  return await intake.reserve(token, operationKey, deployment, context, signal);
+}
+
+function unavailable(): AgentRunFacadeError {
+  return new AgentRunFacadeError(
+    "RUNTIME_UNAVAILABLE",
+    "Screenshot Intake 当前不可用",
+    503,
+  );
+}

package/template/apps/server/src/agent/unavailable-routes.ts

@@ -12,11 +12,19 @@ const unavailableRoutes = [
   ["GET", "/speech/runs/:runId/artifact"],
   ["POST", "/speech/runs/:runId/cancel"],
   ["POST", "/embeddings"],
+  ["POST", "/embeddings/runs"],
+  ["GET", "/embeddings/runs/:runId/result"],
+  ["GET", "/embeddings/runs/:runId/events"],
+  ["POST", "/embeddings/runs/:runId/cancel"],
+  ["GET", "/internal/embeddings/runs/:runId/artifacts/:itemId"],
   ["POST", "/agent/runs"],
   ["GET", "/agent/runs/:runId/events"],
   ["GET", "/agent/runs/:runId/result"],
   ["POST", "/agent/runs/:runId/cancel"],
   ["POST", "/agent/runs/:runId/attach"],
+  ["POST", "/agent/screenshots"],
+  ["PUT", "/agent/screenshots/:token"],
+  ["POST", "/agent/screenshots/:token/cancel"],
   ["POST", "/internal/agent/tools/execute"],
   ["GET", "/system/ai-runtime/summary"],
   ["GET", "/system/ai-runtime/runs"],
@@ -32,6 +40,7 @@ export function registerUnavailableAiRoutes(app: RouteApp): void {
     );
   for (const [method, path] of unavailableRoutes) {
     if (method === "GET") app.get(path, unavailable);
+    else if (method === "PUT") app.put(path, unavailable);
     else app.post(path, unavailable);
   }
 }

package/template/apps/server/src/agent/vision-facade.test.ts

@@ -1,5 +1,10 @@
 import { describe, expect, test } from "bun:test";
 import type { ModuleManifest } from "@southwind-ai/modules";
+import {
+  projectAiOperationReadinessV1,
+  type AiOperationReadinessRequirementV1,
+} from "@southwind-ai/server-sdk";
+import fixture from "@southwind-ai/server-sdk/contracts/ai-runtime-readiness/v1/vlm.json";
 import { featureKey } from "@southwind-ai/shared";
 import type { AiArtifactRuntime } from "../ai-artifact/runtime.js";
 import { buildScopeContext } from "../data-access/context.js";
@@ -14,6 +19,27 @@ import type {
 import { AgentRunFacade } from "./run-facade.js";
 
 describe("Agent Vision facade", () => {
+  test("共享 V1 VLM fixture 能投影为 ready", () => {
+    const now = new Date();
+    const checkedAt = now.toISOString();
+    const snapshot = structuredClone(fixture.snapshot);
+    snapshot.observedAt = checkedAt;
+    snapshot.downstreamProbe = {
+      ...snapshot.downstreamProbe,
+      checkedAt,
+      validUntil: new Date(now.getTime() + 60_000).toISOString(),
+    };
+    expect(
+      projectAiOperationReadinessV1(
+        fixture.operation as unknown as AiOperationReadinessRequirementV1,
+        snapshot,
+        {
+          now: checkedAt,
+        },
+      ).status,
+    ).toBe("ready");
+  });
+
   test("轮换 Artifact 并传递能力、模态和结构化输出 Schema", async () => {
     const module = visionManifest();
     const remote = new FakeRuntime();
@@ -73,8 +99,8 @@ describe("Agent Vision facade", () => {
       outputSchemaKey: "business.output.v1",
       outputSchema: { type: "object" },
       limits: {
-        maxArtifacts: 4,
-        maxPixelsPerArtifact: 20_000_000,
+        maxArtifacts: 16,
+        maxPixelsPerArtifact: 100_000_000,
       },
     });
   });
@@ -113,7 +139,7 @@ class FakeRuntime implements AgentRuntimePort {
   }
 
   health() {
-    return Promise.resolve(new Response());
+    return Promise.resolve(Response.json(readinessSnapshot()));
   }
 }
 
@@ -242,11 +268,27 @@ function visionManifest(): ModuleManifest {
         },
         toolNetworkPolicy: { mode: "forbidden" },
         limits: {
-          maxDurationSeconds: 60,
-          maxInputBytes: 4096,
-          maxOutputBytes: 4096,
-          maxArtifacts: 4,
-          maxPixelsPerArtifact: 20_000_000,
+          maxDurationSeconds: 300,
+          maxInputBytes: 67_108_864,
+          maxOutputBytes: 2_097_152,
+          maxArtifacts: 16,
+          maxPixelsPerArtifact: 100_000_000,
+        },
+        readiness: {
+          contractVersion: 1,
+          expectedDeployment: {
+            key: "ai.openai-compatible",
+            identity:
+              "windy-deployment-v1_f31e8a65-92cb-4d03-8a77-3c1b5e9d4f04",
+            releaseDigest: `sha256:${"d".repeat(64)}`,
+            offersDigest:
+              "sha256:7beb2eeb442fd68c90d94b9469db732acdc51222e152ae0be3ae7be00d2459b2",
+          },
+          variant: "vision",
+          inputModalities: ["text", "image"],
+          outputModalities: ["text"],
+          minConcurrentRuns: 1,
+          minQueueDepth: 0,
         },
       },
     ],
@@ -254,3 +296,51 @@ function visionManifest(): ModuleManifest {
     auditActions: ["business.ai.execute"],
   };
 }
+
+function readinessSnapshot() {
+  const checkedAt = new Date();
+  const observedAt = checkedAt.toISOString();
+  const validUntil = new Date(checkedAt.getTime() + 60_000).toISOString();
+  const deployment = {
+    key: "ai.openai-compatible",
+    identity: "windy-deployment-v1_f31e8a65-92cb-4d03-8a77-3c1b5e9d4f04",
+    releaseDigest: `sha256:${"d".repeat(64)}`,
+    offersDigest:
+      "sha256:7beb2eeb442fd68c90d94b9469db732acdc51222e152ae0be3ae7be00d2459b2",
+  };
+  return {
+    schemaVersion: 1,
+    status: "ready",
+    configured: true,
+    runtimeKind: "agent",
+    observedAt,
+    downstreamProbe: {
+      status: "passed",
+      checkedAt: observedAt,
+      validUntil,
+      latencyMs: 1,
+      deploymentIdentity: deployment.identity,
+      releaseDigest: deployment.releaseDigest,
+      offersDigest: deployment.offersDigest,
+    },
+    deployment,
+    offers: [
+      {
+        variant: "vision",
+        capabilities: ["text-generation", "vision"],
+        inputModalities: ["text", "image"],
+        outputModalities: ["text"],
+        limits: {
+          maxDurationSeconds: 300,
+          maxInputBytes: 67_108_864,
+          maxOutputBytes: 2_097_152,
+          maxArtifacts: 16,
+          maxPixelsPerArtifact: 100_000_000,
+          maxConcurrentRuns: 1,
+          maxQueueDepth: 0,
+        },
+      },
+    ],
+    unavailableReasons: [],
+  };
+}

package/template/apps/server/src/agent/vision-input.ts

@@ -84,8 +84,11 @@ export function validateOutputSchema(
   }
 }
 
-function validateVisionArtifacts(
-  artifacts: readonly ModuleAiArtifactRef[],
+export function validateVisionArtifacts(
+  artifacts: readonly Pick<
+    ModuleAiArtifactRef,
+    "mediaType" | "byteSize" | "sha256"
+  >[],
   definition: ModuleAiOperationDefinition,
 ) {
   if (!artifacts.length) return;

package/template/apps/server/src/agent/vision-readiness-pin.ts

@@ -0,0 +1,74 @@
+import {
+  parseAiRuntimeReadinessV1,
+  type AiRuntimeDeploymentV1,
+} from "@southwind-ai/server-sdk";
+import { AgentRuntimeError } from "./run-contracts.js";
+
+export function sameVisionReadinessDeployment(
+  actual: AiRuntimeDeploymentV1,
+  expected: AiRuntimeDeploymentV1,
+): boolean {
+  return (
+    actual.key === expected.key &&
+    actual.identity === expected.identity &&
+    actual.releaseDigest === expected.releaseDigest &&
+    actual.offersDigest === expected.offersDigest
+  );
+}
+
+export function unavailableVisionReadiness(): AgentRuntimeError {
+  return new AgentRuntimeError(
+    "RUNTIME_UNAVAILABLE",
+    "VLM Runtime 当前不可用",
+    true,
+  );
+}
+
+export async function assertVisionReadinessPin(
+  expected: AiRuntimeDeploymentV1,
+  health: () => Promise<Response>,
+): Promise<void> {
+  const response = await boundedHealth(health);
+  if (response.status !== 200) throw unavailableVisionReadiness();
+  if (!response.headers.get("content-type")?.includes("application/json")) {
+    throw unavailableVisionReadiness();
+  }
+  const length = Number(response.headers.get("content-length") ?? "0");
+  if (!Number.isSafeInteger(length) || length < 0 || length > 64 * 1_024) {
+    throw unavailableVisionReadiness();
+  }
+  let snapshot: unknown;
+  try {
+    const text = await response.text();
+    if (new TextEncoder().encode(text).byteLength > 64 * 1_024) {
+      throw new Error("health body too large");
+    }
+    snapshot = JSON.parse(text);
+  } catch {
+    throw unavailableVisionReadiness();
+  }
+  const parsed = parseAiRuntimeReadinessV1(snapshot, new Date().toISOString());
+  if (
+    !parsed.ok ||
+    parsed.value.status !== "ready" ||
+    !sameVisionReadinessDeployment(parsed.value.deployment, expected)
+  ) {
+    throw unavailableVisionReadiness();
+  }
+}
+
+async function boundedHealth(
+  health: () => Promise<Response>,
+): Promise<Response> {
+  let timeout: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      health(),
+      new Promise<Response>((_, reject) => {
+        timeout = setTimeout(() => reject(unavailableVisionReadiness()), 5_000);
+      }),
+    ]);
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}