create-windy 0.6.14 → 0.6.15

package/template/packages/server-sdk/src/workflow-input-artifact-validation.ts

@@ -0,0 +1,347 @@
+import type { JsonValue } from "@southwind-ai/shared";
+import type {
+  ModuleWorkflowInputJsonArtifactV1,
+  ModuleWorkflowInputMaterializationSourceV1,
+  ModuleWorkflowInputResourceArtifactV1,
+} from "./workflow-input-artifact-port.js";
+
+export const MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1 = {
+  sources: 32,
+  totalBytes: 16 * 1024 * 1024,
+  jsonDepth: 32,
+  jsonNodes: 50_000,
+  arrayItems: 4_096,
+  objectProperties: 2_048,
+  keyBytes: 256,
+  stringBytes: 4 * 1024 * 1024,
+  totalStringBytes: 12 * 1024 * 1024,
+} as const;
+
+const unsafeKeys = new Set(["__proto__", "constructor", "prototype"]);
+const encoder = new TextEncoder();
+
+interface Budget {
+  nodes: number;
+  stringBytes: number;
+  readonly seen: WeakSet<object>;
+}
+
+export function parseModuleWorkflowInputMaterializationBeginV1(value: unknown) {
+  const input = strictRecord(value, ["idempotencyKey", "sources"]);
+  const budget = freshBudget();
+  const result = {
+    idempotencyKey: text(input.idempotencyKey, 200, budget),
+    sources: sources(input.sources, budget),
+  };
+  assertTotalBytes(result);
+  return result;
+}
+
+export function parseModuleWorkflowInputMaterializationPayloadV1(
+  value: unknown,
+) {
+  const input = strictRecord(value, ["schemaVersion", "sources"]);
+  if (input.schemaVersion !== 1) fail();
+  const budget = freshBudget();
+  const result = {
+    schemaVersion: 1 as const,
+    sources: sources(input.sources, budget),
+  };
+  assertTotalBytes(result);
+  return result;
+}
+
+export function parseModuleWorkflowInputMaterializationSourceV1(
+  value: unknown,
+): ModuleWorkflowInputMaterializationSourceV1 {
+  const budget = freshBudget();
+  const result = source(value, budget);
+  assertTotalBytes(result);
+  return result;
+}
+
+export function parseModuleWorkflowInputResourceArtifactV1(
+  value: unknown,
+): ModuleWorkflowInputResourceArtifactV1 {
+  const budget = freshBudget();
+  const result = resourceArtifact(value, budget, false);
+  assertTotalBytes(result);
+  return result;
+}
+
+export function parseModuleWorkflowInputJsonArtifactV1(
+  value: unknown,
+): ModuleWorkflowInputJsonArtifactV1 {
+  const budget = freshBudget();
+  const result = jsonArtifact(value, budget, false);
+  assertTotalBytes(result);
+  return result;
+}
+
+export function parseModuleWorkflowInputJsonValueV1(value: unknown): JsonValue {
+  const budget = freshBudget();
+  const result = json(value, 0, budget);
+  assertTotalBytes(result);
+  return result;
+}
+
+function sources(value: unknown, budget: Budget) {
+  if (
+    !Array.isArray(value) ||
+    value.length < 1 ||
+    value.length > MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.sources
+  ) {
+    fail();
+  }
+  return strictArray(value, value.length).map((item) => source(item, budget));
+}
+
+function source(
+  value: unknown,
+  budget: Budget,
+): ModuleWorkflowInputMaterializationSourceV1 {
+  const mode = dataProperty(value, "mode");
+  if (mode === "resource") {
+    return { mode, ...resourceArtifact(value, budget, true) };
+  }
+  if (mode === "json") return { mode, ...jsonArtifact(value, budget, true) };
+  return fail();
+}
+
+function resourceArtifact(value: unknown, budget: Budget, withMode: boolean) {
+  const input = artifactRecord(value, withMode, "resource");
+  const resource = strictRecord(input.resource, [
+    "fileId",
+    "purpose",
+    "resourceType",
+    "resourceId",
+  ]);
+  return {
+    ...publication(input, budget),
+    resource: {
+      fileId: text(resource.fileId, 256, budget),
+      purpose: text(resource.purpose, 160, budget),
+      resourceType: text(resource.resourceType, 160, budget),
+      resourceId: text(resource.resourceId, 256, budget),
+    },
+  } satisfies ModuleWorkflowInputResourceArtifactV1;
+}
+
+function jsonArtifact(value: unknown, budget: Budget, withMode: boolean) {
+  const input = artifactRecord(value, withMode, "json");
+  return {
+    ...publication(input, budget),
+    value: json(input.value, 0, budget),
+  } satisfies ModuleWorkflowInputJsonArtifactV1;
+}
+
+function artifactRecord(value: unknown, withMode: boolean, mode: string) {
+  const common = [
+    "publicationKey",
+    "kind",
+    "filename",
+    "retentionPolicyKey",
+    "source",
+  ];
+  const specific = mode === "resource" ? "resource" : "value";
+  return strictRecord(
+    value,
+    withMode ? ["mode", ...common, specific] : [...common, specific],
+  );
+}
+
+function publication(input: Record<string, unknown>, budget: Budget) {
+  const source = strictRecord(input.source, [
+    "kind",
+    "id",
+    "version",
+    "sequence",
+    "digest?",
+  ]);
+  if (!Number.isSafeInteger(source.sequence) || Number(source.sequence) < 1)
+    fail();
+  const filename = text(input.filename, 240, budget);
+  if (filename.includes("/") || filename.includes("\\")) fail();
+  return {
+    publicationKey: text(input.publicationKey, 200, budget),
+    kind: text(input.kind, 160, budget),
+    filename,
+    retentionPolicyKey: text(input.retentionPolicyKey, 256, budget),
+    source: {
+      kind: text(source.kind, 160, budget),
+      id: text(source.id, 256, budget),
+      version: text(source.version, 256, budget),
+      sequence: Number(source.sequence),
+      ...(source.digest === undefined
+        ? {}
+        : { digest: digest(source.digest, budget) }),
+    },
+  };
+}
+
+function json(value: unknown, depth: number, budget: Budget): JsonValue {
+  budget.nodes += 1;
+  if (
+    budget.nodes > MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.jsonNodes ||
+    depth > MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.jsonDepth
+  ) {
+    fail();
+  }
+  if (value === null || typeof value === "boolean") return value;
+  if (typeof value === "string") return jsonString(value, budget);
+  if (typeof value === "number") return Number.isFinite(value) ? value : fail();
+  if (!value || typeof value !== "object" || budget.seen.has(value)) fail();
+  budget.seen.add(value);
+  if (Array.isArray(value)) {
+    if (
+      value.length > MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.arrayItems
+    ) {
+      fail();
+    }
+    return strictArray(value, value.length).map((item) =>
+      json(item, depth + 1, budget),
+    );
+  }
+  if (Object.getPrototypeOf(value) !== Object.prototype) fail();
+  const descriptors = Object.getOwnPropertyDescriptors(value);
+  const keys = Reflect.ownKeys(descriptors);
+  if (
+    keys.length >
+      MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.objectProperties ||
+    keys.some((key) => typeof key !== "string" || unsafeKeys.has(key))
+  ) {
+    fail();
+  }
+  const result: Record<string, JsonValue> = {};
+  for (const key of keys as string[]) {
+    keyText(key, budget);
+    const descriptor = descriptors[key];
+    if (!descriptor?.enumerable || !Object.hasOwn(descriptor, "value")) fail();
+    result[key] = json(descriptor.value, depth + 1, budget);
+  }
+  return result;
+}
+
+function strictRecord(value: unknown, keys: readonly string[]) {
+  if (
+    !value ||
+    typeof value !== "object" ||
+    Array.isArray(value) ||
+    Object.getPrototypeOf(value) !== Object.prototype
+  ) {
+    fail();
+  }
+  const descriptors = Object.getOwnPropertyDescriptors(value);
+  const actual = Reflect.ownKeys(descriptors);
+  const required = keys.filter((key) => !key.endsWith("?"));
+  const allowed = new Set(keys.map((key) => key.replace(/\?$/u, "")));
+  if (
+    actual.some((key) => typeof key !== "string" || !allowed.has(key)) ||
+    required.some((key) => !actual.includes(key))
+  ) {
+    fail();
+  }
+  const result: Record<string, unknown> = {};
+  for (const key of actual as string[]) {
+    const descriptor = descriptors[key];
+    if (!descriptor?.enumerable || !Object.hasOwn(descriptor, "value")) fail();
+    result[key] = descriptor.value;
+  }
+  return result;
+}
+
+function dataProperty(value: unknown, key: string) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) fail();
+  const descriptor = Object.getOwnPropertyDescriptor(value, key);
+  if (!descriptor?.enumerable || !Object.hasOwn(descriptor, "value")) fail();
+  return descriptor.value;
+}
+
+function strictArray(value: readonly unknown[], expectedLength: number) {
+  if (Object.getPrototypeOf(value) !== Array.prototype) fail();
+  const descriptors = Object.getOwnPropertyDescriptors(value);
+  const keys = Reflect.ownKeys(descriptors);
+  const lengthDescriptor = Object.getOwnPropertyDescriptor(value, "length");
+  if (
+    keys.length !== expectedLength + 1 ||
+    !lengthDescriptor ||
+    !Object.hasOwn(lengthDescriptor, "value") ||
+    lengthDescriptor.value !== expectedLength ||
+    lengthDescriptor.enumerable
+  ) {
+    fail();
+  }
+  const result: unknown[] = [];
+  for (let index = 0; index < expectedLength; index += 1) {
+    const descriptor = descriptors[String(index)];
+    if (!descriptor?.enumerable || !Object.hasOwn(descriptor, "value")) fail();
+    result.push(descriptor.value);
+  }
+  return result;
+}
+
+function text(value: unknown, maxBytes: number, budget: Budget) {
+  if (
+    typeof value !== "string" ||
+    value !== value.trim() ||
+    !value ||
+    value.includes("\0")
+  )
+    fail();
+  const bytes = encoder.encode(value).byteLength;
+  if (bytes > maxBytes) fail();
+  budget.stringBytes += bytes;
+  if (
+    budget.stringBytes >
+    MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.totalStringBytes
+  )
+    fail();
+  return value;
+}
+
+function jsonString(value: string, budget: Budget) {
+  const bytes = encoder.encode(value).byteLength;
+  if (bytes > MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.stringBytes)
+    fail();
+  budget.stringBytes += bytes;
+  if (
+    budget.stringBytes >
+    MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.totalStringBytes
+  )
+    fail();
+  return value;
+}
+
+function keyText(value: string, budget: Budget) {
+  if (!value || value.includes("\0") || unsafeKeys.has(value)) fail();
+  return text(
+    value,
+    MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.keyBytes,
+    budget,
+  );
+}
+
+function digest(value: unknown, budget: Budget) {
+  const result = text(value, 71, budget);
+  if (!/^sha256:[0-9a-f]{64}$/u.test(result)) fail();
+  return result;
+}
+
+function assertTotalBytes(value: unknown) {
+  const encoded = JSON.stringify(value);
+  if (
+    encoded === undefined ||
+    encoder.encode(encoded).byteLength >
+      MODULE_WORKFLOW_INPUT_MATERIALIZATION_LIMITS_V1.totalBytes
+  ) {
+    fail();
+  }
+}
+
+function freshBudget(): Budget {
+  return { nodes: 0, stringBytes: 0, seen: new WeakSet() };
+}
+
+function fail(): never {
+  throw new Error("WORKFLOW_INPUT_MATERIALIZATION_PAYLOAD_INVALID");
+}

package/template/packages/server-sdk/src/workflow-input-materialization-handler.ts

@@ -0,0 +1,43 @@
+import type { AuditActionType } from "@southwind-ai/shared";
+import type { ModuleDurableHandlerDefinition } from "./durable-handler.js";
+import {
+  moduleWorkflowInputMaterializationHandlerKey,
+  requireModuleWorkflowInputArtifactPublicationPortV1,
+  type ModuleWorkflowInputMaterializationSourceV1,
+} from "./workflow-input-artifact-port.js";
+import { parseModuleWorkflowInputMaterializationPayloadV1 } from "./workflow-input-artifact-validation.js";
+
+export function defineModuleWorkflowInputMaterializationHandlerV1(input: {
+  readonly moduleName: string;
+  readonly featureKey: string;
+  readonly permissionKey: string;
+  readonly auditAction: AuditActionType;
+}): ModuleDurableHandlerDefinition<{
+  readonly schemaVersion: 1;
+  readonly sources: readonly ModuleWorkflowInputMaterializationSourceV1[];
+}> {
+  return {
+    key: moduleWorkflowInputMaterializationHandlerKey(input.moduleName),
+    featureKey: input.featureKey,
+    permissionKey: input.permissionKey,
+    auditAction: input.auditAction,
+    payload: { parse: parsePayload },
+    async execute(payload, context) {
+      const artifacts = requireModuleWorkflowInputArtifactPublicationPortV1(
+        context.workflowInputArtifacts,
+      );
+      for (const source of payload.sources) {
+        if (context.signal.aborted) throw context.signal.reason;
+        if (source.mode === "resource") {
+          const { mode: _mode, ...publication } = source;
+          await artifacts.publishResource(publication);
+        } else {
+          const { mode: _mode, ...publication } = source;
+          await artifacts.publishJson(publication);
+        }
+      }
+    },
+  };
+}
+
+const parsePayload = parseModuleWorkflowInputMaterializationPayloadV1;

package/template/packages/storage/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@southwind-ai/storage",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "license": "UNLICENSED",
   "type": "module",
   "engines": {

package/template/packages/storage/src/blob-store-put-once.test.ts

@@ -0,0 +1,70 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { artifactRef, BlobStoreError } from "./blob-store.js";
+import { LocalFileBlobStore } from "./local-file-blob-store.js";
+import { MemoryBlobStore } from "./memory-blob-store.js";
+
+const roots: string[] = [];
+const stableRef = artifactRef(`blob_${"1".repeat(32)}`);
+
+afterEach(async () => {
+  await Promise.all(
+    roots.splice(0).map((root) => rm(root, { recursive: true })),
+  );
+});
+
+describe("BlobStore putOnce", () => {
+  test("Memory exact replay 返回同一 ref，内容漂移 fail closed", async () => {
+    const store = new MemoryBlobStore();
+    const first = await store.putOnce!(input("same"));
+    const replay = await store.putOnce!(input("same"));
+    expect(replay.ref).toBe(first.ref);
+    await expect(store.putOnce!(input("drift"))).rejects.toMatchObject({
+      code: "BLOB_IDENTITY_CONFLICT",
+    } satisfies Partial<BlobStoreError>);
+  });
+
+  test("Local 并发 exact replay 只发布一个稳定 Blob", async () => {
+    const store = await localStore();
+    const [left, right] = await Promise.all([
+      store.putOnce!(input("same")),
+      store.putOnce!(input("same")),
+    ]);
+    expect(left.ref).toBe(stableRef);
+    expect(right.ref).toBe(stableRef);
+  });
+
+  test("Local blob-only commit unknown 可按同内容补 seal", async () => {
+    const { store, root } = await localStoreWithRoot();
+    const directory = join(root, "blobs", stableRef.slice(5, 7));
+    await mkdir(directory, { recursive: true });
+    await writeFile(join(directory, `${stableRef}.blob`), "same");
+    expect((await store.putOnce!(input("same"))).ref).toBe(stableRef);
+  });
+});
+
+function input(value: string) {
+  const bytes = new TextEncoder().encode(value);
+  return {
+    ref: stableRef,
+    body: new Blob([bytes]).stream(),
+    expectedByteSize: bytes.byteLength,
+    expectedContentHash: {
+      algorithm: "sha256" as const,
+      value: createHash("sha256").update(bytes).digest("hex"),
+    },
+  };
+}
+
+async function localStore() {
+  return (await localStoreWithRoot()).store;
+}
+
+async function localStoreWithRoot() {
+  const root = await mkdtemp(join(tmpdir(), "windy-blob-once-"));
+  roots.push(root);
+  return { root, store: new LocalFileBlobStore({ root }) };
+}

package/template/packages/storage/src/blob-store.ts

@@ -22,8 +22,16 @@ export interface PutBlobInput {
   expectedContentHash?: ContentHash;
 }
 
+export interface PutBlobOnceInput extends PutBlobInput {
+  readonly ref: ArtifactRef;
+  readonly expectedByteSize: number;
+  readonly expectedContentHash: ContentHash;
+}
+
 export interface BlobStore {
   put(input: PutBlobInput): Promise<StoredBlob>;
+  /** Exact replay returns the original Blob; coordinate or content drift fails closed. */
+  putOnce?(input: PutBlobOnceInput): Promise<StoredBlob>;
   inspect(ref: ArtifactRef): Promise<StoredBlob | undefined>;
   open(ref: ArtifactRef): Promise<ReadableBlob | undefined>;
   delete(ref: ArtifactRef): Promise<boolean>;
@@ -42,6 +50,7 @@ export type BlobStoreErrorCode =
   | "INVALID_CONTENT_HASH"
   | "SIZE_MISMATCH"
   | "HASH_MISMATCH"
+  | "BLOB_IDENTITY_CONFLICT"
   | "CORRUPT_BLOB_METADATA";
 
 export class BlobStoreError extends Error {

package/template/packages/storage/src/local-file-blob-put-once.ts

@@ -0,0 +1,120 @@
+import { link, open, stat, type FileHandle } from "node:fs/promises";
+import {
+  BlobStoreError,
+  type ArtifactRef,
+  type PutBlobOnceInput,
+  type StoredBlob,
+} from "./blob-store.js";
+import { createSha256Accumulator } from "./content-hash.js";
+import { writeLocalBlob } from "./local-file-blob-writer.js";
+
+export async function installBlobOnce(input: {
+  ref: ArtifactRef;
+  input: PutBlobOnceInput;
+  blob: string;
+  temporaryBlob: string;
+  now: () => Date;
+}): Promise<StoredBlob> {
+  if (await pathExists(input.blob)) return inspectUnsealedBlob(input);
+  const candidate = await writeLocalBlob(
+    input.temporaryBlob,
+    input.ref,
+    input.input,
+    input.now,
+  );
+  try {
+    await link(input.temporaryBlob, input.blob);
+    return candidate;
+  } catch (error) {
+    if (!isExists(error)) throw error;
+    return inspectUnsealedBlob(input);
+  }
+}
+
+export async function linkOnce(source: string, target: string) {
+  try {
+    await link(source, target);
+  } catch (error) {
+    if (!isExists(error)) throw error;
+  }
+}
+
+export function requireExactBlob(
+  stored: StoredBlob,
+  input: PutBlobOnceInput,
+): StoredBlob {
+  if (
+    stored.ref !== input.ref ||
+    stored.byteSize !== input.expectedByteSize ||
+    stored.contentHash.algorithm !== input.expectedContentHash.algorithm ||
+    stored.contentHash.value !== input.expectedContentHash.value
+  ) {
+    throw new BlobStoreError(
+      "BLOB_IDENTITY_CONFLICT",
+      `Blob 稳定坐标冲突：${input.ref}`,
+    );
+  }
+  return stored;
+}
+
+async function inspectUnsealedBlob(input: {
+  ref: ArtifactRef;
+  input: PutBlobOnceInput;
+  blob: string;
+  now: () => Date;
+}) {
+  const handle = await open(input.blob, "r");
+  try {
+    return requireExactBlob(
+      {
+        ref: input.ref,
+        byteSize: (await handle.stat()).size,
+        contentHash: { algorithm: "sha256", value: await hashHandle(handle) },
+        createdAt: input.now().toISOString(),
+      },
+      input.input,
+    );
+  } finally {
+    await handle.close();
+  }
+}
+
+async function hashHandle(handle: FileHandle): Promise<string> {
+  const hash = createSha256Accumulator();
+  const chunk = new Uint8Array(64 * 1024);
+  let position = 0;
+  while (true) {
+    const { bytesRead } = await handle.read(
+      chunk,
+      0,
+      chunk.byteLength,
+      position,
+    );
+    if (bytesRead === 0) break;
+    hash.update(chunk.subarray(0, bytesRead));
+    position += bytesRead;
+  }
+  return hash.digest().value;
+}
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch (error) {
+    if (isCode(error, "ENOENT")) return false;
+    throw error;
+  }
+}
+
+function isExists(error: unknown) {
+  return isCode(error, "EEXIST");
+}
+
+function isCode(error: unknown, code: string) {
+  return (
+    error instanceof Error &&
+    "code" in error &&
+    (error as NodeJS.ErrnoException).code === code
+  );
+}

package/template/packages/storage/src/local-file-blob-store.ts

@@ -15,6 +15,7 @@ import {
   type ArtifactRef,
   type BlobStore,
   type PutBlobInput,
+  type PutBlobOnceInput,
   type ReadableBlob,
   type StoredBlob,
 } from "./blob-store.js";
@@ -23,6 +24,11 @@ import {
   validateExpectedSize,
   writeLocalBlob,
 } from "./local-file-blob-writer.js";
+import {
+  installBlobOnce,
+  linkOnce,
+  requireExactBlob,
+} from "./local-file-blob-put-once.js";
 
 export interface LocalFileBlobStoreOptions {
   root: string;
@@ -40,8 +46,54 @@ export class LocalFileBlobStore implements BlobStore {
   }
 
   async put(input: PutBlobInput): Promise<StoredBlob> {
+    return this.write(this.createRef(), input);
+  }
+
+  async putOnce(input: PutBlobOnceInput): Promise<StoredBlob> {
+    const ref = artifactRef(input.ref);
+    validateExpectedSize(input.expectedByteSize);
+    const paths = this.paths(ref);
+    await mkdir(paths.directory, { recursive: true });
+    await mkdir(paths.temporaryDirectory, { recursive: true });
+    const suffix = randomUUID();
+    const temporaryBlob = join(
+      paths.temporaryDirectory,
+      `${ref}.${suffix}.partial`,
+    );
+    const temporaryMetadata = join(
+      paths.temporaryDirectory,
+      `${ref}.${suffix}.json.partial`,
+    );
+    try {
+      const existing = await this.inspect(ref);
+      if (existing) return requireExactBlob(existing, input);
+      const stored = await installBlobOnce({
+        ref,
+        input,
+        blob: paths.blob,
+        temporaryBlob,
+        now: this.now,
+      });
+      await syncDirectory(paths.directory);
+      await writeDurably(temporaryMetadata, `${JSON.stringify(stored)}\n`);
+      await linkOnce(temporaryMetadata, paths.metadata);
+      await syncDirectory(paths.directory);
+      const committed = await this.inspect(ref);
+      if (!committed) throw corruptMetadata(ref);
+      return requireExactBlob(committed, input);
+    } finally {
+      await Promise.all([
+        rm(temporaryBlob, { force: true }),
+        rm(temporaryMetadata, { force: true }),
+      ]);
+    }
+  }
+
+  private async write(
+    ref: ArtifactRef,
+    input: PutBlobInput,
+  ): Promise<StoredBlob> {
     validateExpectedSize(input.expectedByteSize);
-    const ref = this.createRef();
     const paths = this.paths(ref);
     await mkdir(paths.directory, { recursive: true });
     await mkdir(paths.temporaryDirectory, { recursive: true });