@world-engines/project-host 0.1.0-alpha.0

package/dist/embedding-cache.js

@@ -0,0 +1,317 @@
+import { createHash, randomUUID } from "node:crypto";
+import { open, readFile, rename, rm } from "node:fs/promises";
+import { basename, dirname, join } from "node:path";
+import { EMBEDDING_BATCH_SIZE } from "./authoring-model-connection.js";
+import { assertSafeExistingProjectPath, assertSafeProjectParent, ensureSafeProjectDirectory, safeProjectPath, } from "./safe-project-path.js";
+export class EmbeddingCacheError extends Error {
+    code;
+    name = "EmbeddingCacheError";
+    constructor(code) {
+        super(`embedding_cache:${code}`);
+        this.code = code;
+    }
+}
+const inFlight = new Map();
+const directoryFlights = new Map();
+const CACHE_IO_CONCURRENCY = 16;
+const EMBEDDING_BATCH_CONCURRENCY = 4;
+// 发生错误后停止领取新任务，但已发出的请求仍须完成校验和持久化。
+async function runBounded(count, concurrency, task) {
+    let next = 0;
+    let failed = false;
+    let failure;
+    await Promise.all(Array.from({ length: Math.min(count, concurrency) }, async () => {
+        while (!failed && next < count) {
+            const index = next++;
+            try {
+                await task(index);
+            }
+            catch (error) {
+                if (!failed) {
+                    failed = true;
+                    failure = error;
+                }
+            }
+        }
+    }));
+    if (failed)
+        throw failure;
+}
+function assertNotAborted(signal) {
+    if (signal?.aborted)
+        fail("aborted");
+}
+function fail(code) {
+    throw new EmbeddingCacheError(code);
+}
+function sha256(value) {
+    return createHash("sha256").update(value, "utf8").digest("hex");
+}
+function canonicalOrigin(value) {
+    if (typeof value !== "string")
+        fail("invalid_identity");
+    try {
+        const url = new URL(value);
+        if (url.username !== "" || url.password !== "" || url.search !== "" || url.hash !== ""
+            || (url.pathname !== "" && url.pathname !== "/") || url.origin === "null")
+            fail("invalid_identity");
+        return url.origin;
+    }
+    catch (error) {
+        if (error instanceof EmbeddingCacheError)
+            throw error;
+        fail("invalid_identity");
+    }
+}
+function identity(options) {
+    const embeddedOrigin = options.embedding.origin;
+    if (options.origin !== undefined && embeddedOrigin !== undefined
+        && canonicalOrigin(options.origin) !== canonicalOrigin(embeddedOrigin))
+        fail("invalid_identity");
+    const origin = canonicalOrigin(options.origin ?? embeddedOrigin);
+    if (typeof options.embedding.providerId !== "string" || options.embedding.providerId.length === 0
+        || typeof options.embedding.modelId !== "string" || options.embedding.modelId.length === 0
+        || !Number.isSafeInteger(options.embedding.dimensions) || options.embedding.dimensions <= 0
+        || (options.embedding.descriptorSha256 !== undefined && !/^[a-f0-9]{64}$/u.test(options.embedding.descriptorSha256))
+        || (options.embedding.descriptorSha256ByInputKind !== undefined
+            && (!/^[a-f0-9]{64}$/u.test(options.embedding.descriptorSha256ByInputKind.document)
+                || !/^[a-f0-9]{64}$/u.test(options.embedding.descriptorSha256ByInputKind.query))))
+        fail("invalid_identity");
+    const documentDescriptor = options.embedding.descriptorSha256ByInputKind?.document ?? options.embedding.descriptorSha256 ?? null;
+    const queryDescriptor = options.embedding.descriptorSha256ByInputKind?.query ?? options.embedding.descriptorSha256 ?? null;
+    return Object.freeze({
+        origin,
+        providerId: options.embedding.providerId,
+        modelId: options.embedding.modelId,
+        dimensions: options.embedding.dimensions,
+        descriptorSha256ByInputKind: Object.freeze({ document: documentDescriptor, query: queryDescriptor }),
+    });
+}
+function inputIdentity(base, inputKind) {
+    return Object.freeze({
+        origin: base.origin, providerId: base.providerId, modelId: base.modelId, dimensions: base.dimensions,
+        descriptorSha256: base.descriptorSha256ByInputKind[inputKind], inputKind,
+    });
+}
+function cacheLocation(cacheIdentity) {
+    const identityJson = JSON.stringify({ schema_version: 2, ...cacheIdentity });
+    const identitySha256 = sha256(identityJson);
+    return { identitySha256, relativeRoot: `.worldengine/cache/embeddings/v1/${identitySha256}` };
+}
+function validVector(value, dimensions) {
+    return Array.isArray(value) && value.length === dimensions
+        && Array.from(value).every((entry) => typeof entry === "number" && Number.isFinite(entry));
+}
+/** 只读查询现有 corpus cache；任一 miss 返回 null，绝不调用 provider 或创建目录。 */
+export async function readCachedEmbeddingVectors(options, textSha256s) {
+    const cacheIdentity = inputIdentity(identity(options), options.inputKind ?? "document");
+    const { identitySha256, relativeRoot } = cacheLocation(cacheIdentity);
+    const vectors = [];
+    for (const textSha256 of textSha256s) {
+        if (!/^[a-f0-9]{64}$/u.test(textSha256))
+            fail("invalid_input");
+        const path = await safeProjectPath(options.projectRoot, `${relativeRoot}/${textSha256.slice(0, 2)}/${textSha256}.json`);
+        try {
+            await assertSafeExistingProjectPath(options.projectRoot, path);
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return null;
+            throw error;
+        }
+        let entry;
+        try {
+            entry = JSON.parse(await readFile(path, "utf8"));
+        }
+        catch {
+            fail("corrupt");
+        }
+        if (entry.schema_version !== 1 || entry.identity_sha256 !== identitySha256 || entry.text_sha256 !== textSha256
+            || entry.dimensions !== cacheIdentity.dimensions || !validVector(entry.vector, cacheIdentity.dimensions))
+            fail("corrupt");
+        vectors.push(Object.freeze([...entry.vector]));
+    }
+    return Object.freeze(vectors);
+}
+function deferred() {
+    let resolve;
+    let reject;
+    const promise = new Promise((resolvePromise, rejectPromise) => {
+        resolve = resolvePromise;
+        reject = rejectPromise;
+    });
+    // 所有在途批次落盘后才汇总结果；提前观察拒绝，避免共享 waiter 出现未处理拒绝。
+    void promise.catch(() => { });
+    return { promise, resolve, reject };
+}
+async function ensureDirectory(root, relative) {
+    const key = `${root}\0${relative}`;
+    const pending = directoryFlights.get(key);
+    if (pending !== undefined)
+        return pending;
+    const created = (async () => {
+        try {
+            return await ensureSafeProjectDirectory(root, relative);
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+            return await assertSafeExistingProjectPath(root, join(root, ...relative.split("/")));
+        }
+    })();
+    directoryFlights.set(key, created);
+    try {
+        return await created;
+    }
+    finally {
+        if (directoryFlights.get(key) === created)
+            directoryFlights.delete(key);
+    }
+}
+async function atomicWrite(root, path, bytes) {
+    await assertSafeProjectParent(root, path);
+    const temporary = join(dirname(path), `.${basename(path)}.${randomUUID()}.tmp`);
+    const handle = await open(temporary, "wx");
+    try {
+        await handle.writeFile(bytes, "utf8");
+        await handle.sync();
+    }
+    catch (error) {
+        await handle.close();
+        await rm(temporary, { force: true });
+        throw error;
+    }
+    await handle.close();
+    try {
+        await assertSafeProjectParent(root, path);
+        await rename(temporary, path);
+    }
+    catch (error) {
+        await rm(temporary, { force: true });
+        throw error;
+    }
+}
+export async function createCachedEmbeddingPort(options) {
+    const baseIdentity = identity(options);
+    const counts = { hits: 0, misses: 0, remoteTexts: 0, writes: 0 };
+    const readEntry = async (path, textSha256, cacheIdentity, identitySha256) => {
+        try {
+            await assertSafeExistingProjectPath(options.projectRoot, path);
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return null;
+            throw error;
+        }
+        let entry;
+        try {
+            entry = JSON.parse(await readFile(path, "utf8"));
+        }
+        catch {
+            fail("corrupt");
+        }
+        if (entry.schema_version !== 1 || entry.identity_sha256 !== identitySha256 || entry.text_sha256 !== textSha256
+            || entry.dimensions !== cacheIdentity.dimensions || !validVector(entry.vector, cacheIdentity.dimensions))
+            fail("corrupt");
+        return Object.freeze([...entry.vector]);
+    };
+    const writeEntry = async (path, textSha256, vector, cacheIdentity, identitySha256) => {
+        await atomicWrite(options.projectRoot, path, `${JSON.stringify({
+            schema_version: 1,
+            identity: cacheIdentity,
+            identity_sha256: identitySha256,
+            text_sha256: textSha256,
+            dimensions: cacheIdentity.dimensions,
+            vector,
+        })}\n`);
+        counts.writes += 1;
+    };
+    return Object.freeze({
+        origin: baseIdentity.origin,
+        providerId: baseIdentity.providerId,
+        modelId: baseIdentity.modelId,
+        dimensions: baseIdentity.dimensions,
+        ...(baseIdentity.descriptorSha256ByInputKind.document === null ? {} : { descriptorSha256: baseIdentity.descriptorSha256ByInputKind.document }),
+        ...(baseIdentity.descriptorSha256ByInputKind.document === null || baseIdentity.descriptorSha256ByInputKind.query === null ? {} : {
+            descriptorSha256ByInputKind: Object.freeze({ document: baseIdentity.descriptorSha256ByInputKind.document, query: baseIdentity.descriptorSha256ByInputKind.query }),
+        }),
+        async embed(texts, embedOptions) {
+            if (!Array.isArray(texts) || texts.length === 0 || texts.some((value) => typeof value !== "string" || value.length === 0))
+                fail("invalid_input");
+            assertNotAborted(embedOptions?.signal);
+            const inputKind = embedOptions?.inputKind ?? "document";
+            const cacheIdentity = inputIdentity(baseIdentity, inputKind);
+            const { identitySha256, relativeRoot } = cacheLocation(cacheIdentity);
+            await ensureDirectory(options.projectRoot, relativeRoot);
+            const entryPath = async (textSha256) => {
+                const relativeDirectory = `${relativeRoot}/${textSha256.slice(0, 2)}`;
+                await ensureDirectory(options.projectRoot, relativeDirectory);
+                return safeProjectPath(options.projectRoot, `${relativeDirectory}/${textSha256}.json`);
+            };
+            const unique = new Map();
+            for (const value of texts)
+                unique.set(sha256(value), value);
+            const resolved = new Map();
+            const waits = new Map();
+            const owned = [];
+            try {
+                const entries = [...unique];
+                await runBounded(entries.length, CACHE_IO_CONCURRENCY, async (index) => {
+                    assertNotAborted(embedOptions?.signal);
+                    const [textSha256, value] = entries[index];
+                    const path = await entryPath(textSha256);
+                    const cached = await readEntry(path, textSha256, cacheIdentity, identitySha256);
+                    if (cached !== null) {
+                        counts.hits += 1;
+                        resolved.set(textSha256, cached);
+                        return;
+                    }
+                    counts.misses += 1;
+                    const key = `${path}\0${textSha256}`;
+                    const current = inFlight.get(key);
+                    if (current !== undefined) {
+                        waits.set(textSha256, current);
+                        return;
+                    }
+                    const claim = deferred();
+                    inFlight.set(key, claim.promise);
+                    waits.set(textSha256, claim.promise);
+                    owned.push({ index, textSha256, text: value, path, deferred: claim });
+                });
+                owned.sort((left, right) => left.index - right.index);
+                await runBounded(Math.ceil(owned.length / EMBEDDING_BATCH_SIZE), EMBEDDING_BATCH_CONCURRENCY, async (index) => {
+                    assertNotAborted(embedOptions?.signal);
+                    const batch = owned.slice(index * EMBEDDING_BATCH_SIZE, (index + 1) * EMBEDDING_BATCH_SIZE);
+                    counts.remoteTexts += batch.length;
+                    const vectors = await options.embedding.embed(batch.map((entry) => entry.text), embedOptions);
+                    if (!Array.isArray(vectors) || vectors.length !== batch.length
+                        || Array.from(vectors).some((vector) => !validVector(vector, cacheIdentity.dimensions)))
+                        fail("provider_response_invalid");
+                    const snapshots = vectors.map((vector) => Object.freeze([...vector]));
+                    // 收到成功响应后即使 signal 已取消也提交本批；每个 worker 最多一个 fsync。
+                    for (const [vectorIndex, entry] of batch.entries()) {
+                        const vector = snapshots[vectorIndex];
+                        await writeEntry(entry.path, entry.textSha256, vector, cacheIdentity, identitySha256);
+                        entry.deferred.resolve(vector);
+                        inFlight.delete(`${entry.path}\0${entry.textSha256}`);
+                    }
+                });
+            }
+            catch (error) {
+                for (const entry of owned)
+                    entry.deferred.reject(error);
+                throw error;
+            }
+            finally {
+                for (const entry of owned) {
+                    const key = `${entry.path}\0${entry.textSha256}`;
+                    if (inFlight.get(key) === entry.deferred.promise)
+                        inFlight.delete(key);
+                }
+            }
+            await Promise.all([...waits].map(async ([textSha256, promise]) => { resolved.set(textSha256, await promise); }));
+            return Object.freeze(texts.map((value) => resolved.get(sha256(value))));
+        },
+        getCounts() { return Object.freeze({ ...counts }); },
+    });
+}

package/dist/frozen-review-local-pipeline.d.ts

@@ -0,0 +1,88 @@
+import { type NeutralSourceCompileInput } from "@world-engines/scenario-author-source";
+import type { FrozenArtifactEmbeddingBatch, FrozenPublishInput, NeutralArtifactGraphProjection, PreparedNeutralArtifactEmbeddingSnapshot } from "@world-engines/scenario-review-snapshot";
+import { type CompileNeutralScenarioSourceOutcome, type CompileTargetProfile } from "@world-engines/tmw";
+import type { FrozenReviewCapture, FrozenReviewEmbedding, FrozenReviewLocalPipeline, FrozenReviewRecoveryPort } from "./frozen-review.js";
+import { type FrozenReviewWasmEncryptor } from "./frozen-review-wasm-encryptor.js";
+import type { ProjectTransferTransport } from "./transfer/transport.js";
+import type { AuthoringEmbeddingPort } from "./authoring-model-connection.js";
+export interface FrozenReviewTargetBinding {
+    readonly schema_version: 1;
+    readonly project_id: string;
+    readonly scenario_id: string;
+    readonly source_digest: string;
+    readonly source_head_digest: string;
+}
+export interface FrozenReviewLocalAssemblerInput {
+    readonly capture: FrozenReviewCapture;
+    readonly source: NeutralSourceCompileInput;
+    readonly profile: CompileTargetProfile;
+    readonly compilation: Extract<CompileNeutralScenarioSourceOutcome, {
+        readonly ok: true;
+    }>;
+    readonly embedding: FrozenReviewEmbedding;
+    readonly vectors: ReadonlyMap<string, readonly number[]>;
+    readonly embedding_snapshot: PreparedNeutralArtifactEmbeddingSnapshot;
+    readonly signal?: AbortSignal;
+}
+export interface CreateFrozenReviewLocalArtifactAssemblerOptions {
+    readonly project_root: string;
+    readonly project_id: string;
+    readonly transport: ProjectTransferTransport;
+    readonly crypto?: Pick<FrozenReviewWasmEncryptor, "encrypt"> & Partial<Pick<FrozenReviewWasmEncryptor, "pack">>;
+}
+export declare function createFrozenReviewEmbeddingBatch(embedding: AuthoringEmbeddingPort): FrozenArtifactEmbeddingBatch;
+export interface CreateFrozenReviewRecoveryPortOptions {
+    readonly project_root: string;
+    readonly project_id: string;
+    readonly transport: ProjectTransferTransport;
+    readonly crypto?: Pick<FrozenReviewWasmEncryptor, "encrypt" | "decrypt">;
+}
+/**
+ * Browser 与 Node 必须共用的最后一段 authority。它负责把已编译模型、向量、View build
+ * 和 WASM scnb-v2 delivery 组装为 FrozenPublishInput；Node pipeline 不复制这些规则。
+ */
+export interface FrozenReviewLocalArtifactAssembler {
+    embeddingGraph(input: Readonly<{
+        capture: FrozenReviewCapture;
+        source: NeutralSourceCompileInput;
+        profile: CompileTargetProfile;
+        compilation: Extract<CompileNeutralScenarioSourceOutcome, {
+            readonly ok: true;
+        }>;
+    }>): Promise<NeutralArtifactGraphProjection>;
+    assemble(input: FrozenReviewLocalAssemblerInput): Promise<FrozenPublishInput>;
+}
+export interface CreateFrozenReviewLocalPipelineOptions {
+    readonly project_root: string;
+    readonly project_id: string;
+    readonly run_embedding_batch: FrozenArtifactEmbeddingBatch;
+    /** Host query-only ready adapter 可注入已校验 formal snapshot；返回 null 时才允许本次显式提审调用 provider。 */
+    readonly read_prepared_embedding_snapshot?: (capture: FrozenReviewCapture) => Promise<PreparedNeutralArtifactEmbeddingSnapshot | null>;
+    readonly assembler: FrozenReviewLocalArtifactAssembler;
+    /** WESP owner 提供的正式远端 materialization readback；revision_bytes 是 canonical Neutral protobuf。 */
+    readonly read_source_ready: () => Promise<Readonly<{
+        target: FrozenReviewTargetBinding;
+        content_epoch: number;
+        head: NeutralSourceCompileInput["witness"];
+        revision_bytes: Uint8Array;
+    }>>;
+    readonly admit_profile?: (profile: CompileTargetProfile) => Promise<void>;
+}
+export interface FrozenReviewLocalPipelineWithPreparedEmbedding extends FrozenReviewLocalPipeline {
+    /** 只读；仅返回 embed 步骤已经由 shared module 校验的 formal batch/index。 */
+    getPreparedEmbeddingSnapshot(capture: Pick<FrozenReviewCapture, "scenario_id" | "source_revision" | "revision_digest">): PreparedNeutralArtifactEmbeddingSnapshot | null;
+}
+/** 与 Creator 共读 Edge CompilerReleaseAuthority；transport 持有认证，CLI 不接触 cookie/token。 */
+export declare function createCompilerReleaseProfileAdmission(transport: ProjectTransferTransport, projectId: string): (profile: CompileTargetProfile) => Promise<void>;
+export declare function readFrozenReviewTargetBinding(root: string, expectedProjectId: string): Promise<FrozenReviewTargetBinding>;
+/** Coordinator 的随机密文恢复 port；每次仅短暂消费同 scenario database key。 */
+export declare function createFrozenReviewRecoveryPort(options: CreateFrozenReviewRecoveryPortOptions): FrozenReviewRecoveryPort;
+/**
+ * Standalone Node 的真实发布装配器：remote capture 的同一 compiler outcome 在共享模块物化，
+ * 随后构建双端 View、用 broker key + Rust/WASM 生成 SCNB，再返回 FrozenPublishInput。
+ * prepare 不写 mutable module；唯一远端写由 Coordinator 的 attempt POST 执行。
+ */
+export declare function createFrozenReviewLocalArtifactAssembler(options: CreateFrozenReviewLocalArtifactAssemblerOptions): FrozenReviewLocalArtifactAssembler;
+/** neutral-transfer 在远端 readback 成功后调用；只持久化公开 identity/digest。 */
+export declare function writeFrozenReviewTargetBinding(root: string, binding: FrozenReviewTargetBinding): Promise<FrozenReviewTargetBinding>;
+export declare function createFrozenReviewLocalPipeline(options: CreateFrozenReviewLocalPipelineOptions): FrozenReviewLocalPipelineWithPreparedEmbedding;