@world-engines/project-host 0.1.0-alpha.0

package/dist/document-relation-role.d.ts

@@ -0,0 +1,14 @@
+export type DocumentRelationRole = "structural" | "semantic" | "unknown";
+export interface DocumentRelationRoleInput {
+    readonly relation: Readonly<{
+        kind: string;
+        props: Readonly<Record<string, unknown>>;
+    }>;
+    readonly endpoints: Readonly<{
+        srcKind?: string | undefined;
+        dstKind?: string | undefined;
+    }>;
+}
+export declare function classifyDocumentRelationRole(input: DocumentRelationRoleInput): DocumentRelationRole;
+/** `undefined` 表示端点尚未回读或结构形状不完整；调用方不得把它当成语义边。 */
+export declare function isStructuralRelation(input: DocumentRelationRoleInput): boolean | undefined;

package/dist/document-relation-role.js

@@ -0,0 +1,86 @@
+const RESERVED_RELATION_KINDS = new Set([
+    "part_of",
+    "next_chunk",
+    "mentions",
+    "supports",
+    "describes",
+    "describes_source",
+    "describes_target",
+]);
+const SEMANTIC_ENTITY_KINDS = new Set([
+    "world_document_concept",
+    "world_document_event",
+    "world_document_object",
+    "world_document_organization",
+    "world_document_person",
+    "world_document_place",
+]);
+const DIRECT_EVIDENCE_KEYS = ["end_offset", "offset_semantics", "quote", "spans", "start_offset", "unit_ids"];
+function hasSemanticAssertion(props) {
+    return typeof props.assertion === "string" && props.assertion.trim().length > 0
+        && props.evidence !== null && typeof props.evidence === "object" && !Array.isArray(props.evidence)
+        && Array.isArray(props.source_chunk_ids) && props.source_chunk_ids.length > 0
+        && props.source_chunk_ids.every((id) => typeof id === "string" && id.length > 0);
+}
+function hasExactOrdinal(props) {
+    return Object.keys(props).length === 1 && Number.isSafeInteger(props.ordinal) && props.ordinal >= 0;
+}
+function isEmptyProps(props) {
+    return Object.keys(props).length === 0;
+}
+function hasExactDirectEvidence(props) {
+    const keys = Object.keys(props).sort();
+    if (keys.length !== DIRECT_EVIDENCE_KEYS.length || keys.some((key, index) => key !== [...DIRECT_EVIDENCE_KEYS].sort()[index]))
+        return false;
+    if (typeof props.quote !== "string" || !Number.isSafeInteger(props.start_offset) || !Number.isSafeInteger(props.end_offset)
+        || props.start_offset < 0 || props.end_offset <= props.start_offset
+        || props.offset_semantics !== "envelope_only")
+        return false;
+    if (!Array.isArray(props.unit_ids) || props.unit_ids.length < 1 || !props.unit_ids.every((id) => typeof id === "string" && id.length > 0))
+        return false;
+    return Array.isArray(props.spans) && props.spans.length > 0 && props.spans.every((value) => {
+        if (value === null || typeof value !== "object" || Array.isArray(value))
+            return false;
+        const span = value;
+        return Object.keys(span).sort().join("\0") === "end_offset\0quote\0start_offset\0unit_id"
+            && typeof span.unit_id === "string" && typeof span.quote === "string"
+            && Number.isSafeInteger(span.start_offset) && Number.isSafeInteger(span.end_offset)
+            && span.start_offset >= 0 && span.end_offset > span.start_offset;
+    });
+}
+export function classifyDocumentRelationRole(input) {
+    const kind = input.relation.kind.toLowerCase();
+    if (!RESERVED_RELATION_KINDS.has(kind))
+        return "semantic";
+    const { srcKind, dstKind } = input.endpoints;
+    if (srcKind === undefined || dstKind === undefined)
+        return "unknown";
+    if (hasSemanticAssertion(input.relation.props))
+        return "semantic";
+    if (kind === "part_of" && srcKind === "world_source_chunk" && dstKind === "world_source_document") {
+        return hasExactOrdinal(input.relation.props) ? "structural" : "unknown";
+    }
+    if (kind === "mentions" && srcKind === "world_source_chunk" && SEMANTIC_ENTITY_KINDS.has(dstKind)) {
+        return hasExactDirectEvidence(input.relation.props) ? "structural" : "unknown";
+    }
+    if (kind === "next_chunk" && srcKind === "world_source_chunk" && dstKind === "world_source_chunk") {
+        return isEmptyProps(input.relation.props) ? "structural" : "unknown";
+    }
+    if (kind === "supports" && srcKind === "world_source_chunk"
+        && (dstKind === "world_document_statement" || dstKind === "world_document_edge_assertion")) {
+        return hasExactDirectEvidence(input.relation.props) ? "structural" : "unknown";
+    }
+    if (kind === "describes" && srcKind === "world_document_statement" && SEMANTIC_ENTITY_KINDS.has(dstKind)) {
+        return isEmptyProps(input.relation.props) ? "structural" : "unknown";
+    }
+    if ((kind === "describes_source" || kind === "describes_target")
+        && srcKind === "world_document_edge_assertion" && SEMANTIC_ENTITY_KINDS.has(dstKind)) {
+        return isEmptyProps(input.relation.props) ? "structural" : "unknown";
+    }
+    return "semantic";
+}
+/** `undefined` 表示端点尚未回读或结构形状不完整；调用方不得把它当成语义边。 */
+export function isStructuralRelation(input) {
+    const role = classifyDocumentRelationRole(input);
+    return role === "unknown" ? undefined : role === "structural";
+}

package/dist/document-source.d.ts

@@ -0,0 +1,107 @@
+export declare const DOCUMENT_SOURCE_SCHEMA_VERSION: 1;
+export declare const DOCUMENT_PLAN_SCHEMA_VERSION: 1;
+export type DocumentUnitKind = "control" | "chapter_header" | "sentence" | "layout";
+export type DocumentUnit = Readonly<{
+    id: string;
+    kind: DocumentUnitKind;
+    startOffset: number;
+    endOffset: number;
+    text: string;
+    chapterId?: string;
+}>;
+export type DocumentWindow = Readonly<{
+    id: string;
+    coreUnitIds: readonly string[];
+    contextUnitIds: readonly string[];
+    chapterId?: string;
+    startOffset: number;
+    coreStartOffset: number;
+    coreEndOffset: number;
+    endOffset: number;
+    text: string;
+}>;
+export type DocumentSource = Readonly<{
+    schemaVersion: typeof DOCUMENT_SOURCE_SCHEMA_VERSION;
+    documentId: string;
+    sourceId: string;
+    sourceRevision: string;
+    scopeId: string;
+    sourceSha256: string;
+    byteLength: number;
+    utf16Length: number;
+    originalBytesBase64: string;
+    utf8Bom: "present" | "absent";
+    /** 所有 offset 均以包含 U+FEFF BOM（若存在）的 decoded UTF-16 code unit 为域。 */
+    offsetDomain: "decoded_utf16_including_bom";
+    units: readonly DocumentUnit[];
+    windows: readonly DocumentWindow[];
+}>;
+export type DocumentSourceOptions = Readonly<{
+    targetWindowUtf16?: number;
+    maximumWindowUtf16?: number;
+    maximumAtomicUtf16?: number;
+    includePreviousSentenceContext?: boolean;
+}>;
+export type BuildDocumentSourceInput = Readonly<{
+    bytes: Uint8Array;
+    sourceId: string;
+    sourceRevision: string;
+    scopeId: string;
+    options?: DocumentSourceOptions;
+}>;
+export type DocumentPlan = Readonly<{
+    schemaVersion: typeof DOCUMENT_PLAN_SCHEMA_VERSION;
+    documentId: string;
+    segments: readonly Readonly<{
+        startUnitId: string;
+        endUnitId: string;
+        title: string;
+    }>[];
+}>;
+export type ValidatedDocumentSegment = Readonly<{
+    id: string;
+    startUnitId: string;
+    endUnitId: string;
+    title: string;
+    chapterId?: string;
+    startOffset: number;
+    endOffset: number;
+    indexText: string;
+}>;
+export type ValidatedDocumentPlan = Readonly<{
+    schemaVersion: typeof DOCUMENT_PLAN_SCHEMA_VERSION;
+    documentId: string;
+    segments: readonly ValidatedDocumentSegment[];
+}>;
+export type WorldSourceDocumentNode = Readonly<{
+    kind: "world_source_document";
+    id: string;
+    props: Readonly<{
+        scopeId: string;
+        sourceId: string;
+        sourceRevision: string;
+        sourceSha256: string;
+        byteLength: number;
+        utf16Length: number;
+        utf8Bom: "present" | "absent";
+        offsetDomain: "decoded_utf16_including_bom";
+    }>;
+}>;
+export type WorldSourceChunkNode = Readonly<{
+    kind: "world_source_chunk";
+    id: string;
+    props: Readonly<{
+        documentId: string;
+        ordinal: number;
+        title: string;
+        startUnitId: string;
+        endUnitId: string;
+        startOffset: number;
+        endOffset: number;
+        index_text: string;
+    }>;
+}>;
+/** 深 Module 的唯一构建 Interface：fatal decode、无损校验、结构切分与窗口化一次完成。 */
+export declare function buildDocumentSource(input: BuildDocumentSourceInput): DocumentSource;
+export declare function documentSourceBytes(document: DocumentSource): Uint8Array;
+export declare function validateDocumentPlan(document: DocumentSource, plan: DocumentPlan): ValidatedDocumentPlan;

package/dist/document-source.js

@@ -0,0 +1,255 @@
+import { createHash } from "node:crypto";
+export const DOCUMENT_SOURCE_SCHEMA_VERSION = 1;
+export const DOCUMENT_PLAN_SCHEMA_VERSION = 1;
+// 章节标记必须位于行首，或紧随任意标题与既有斜线分隔符；不依赖文章／书籍名称。
+const HEADER = /^(?:[^\/／\r\n]+[\/／]\s*)?第[0-9〇○零一二三四五六七八九十百千两]+[回章节](?:\s|$)/u;
+// 单个 CRLF 是一个 line break；禁止 regexp 回溯把它误拆成 CR + LF 两个空行。
+const BLANK_SEPARATOR = /(?:\r\n|\r(?!\n)|(?<!\r)\n)[\t \u3000]*(?:\r\n|\r(?!\n)|(?<!\r)\n)+/gu;
+const SENTENCE_SEGMENTER = new Intl.Segmenter("zh-Hans", { granularity: "sentence" });
+const GRAPHEME_SEGMENTER = new Intl.Segmenter("zh-Hans", { granularity: "grapheme" });
+function invariant(condition, code) {
+    if (!condition)
+        throw new Error(code);
+}
+function identifier(value, code) {
+    invariant(typeof value === "string" && value.length > 0 && value.length <= 512, code);
+    invariant(value.trim() === value && value.normalize("NFC") === value && !/[\0\r\n]/u.test(value), code);
+    return value;
+}
+function sha256(bytes) {
+    return createHash("sha256").update(bytes).digest("hex");
+}
+function equalBytes(left, right) {
+    if (left.byteLength !== right.byteLength)
+        return false;
+    return left.every((value, index) => value === right[index]);
+}
+function chapterHeaders(text) {
+    const headers = [];
+    let start = text.charCodeAt(0) === 0xfeff ? 1 : 0;
+    while (start < text.length) {
+        const newline = text.indexOf("\n", start);
+        let end = newline < 0 ? text.length : newline;
+        if (end > start && text.charCodeAt(end - 1) === 0x0d)
+            end -= 1;
+        const line = text.slice(start, end);
+        if (HEADER.test(line.trimStart()))
+            headers.push({ start, end });
+        if (newline < 0)
+            break;
+        start = newline + 1;
+    }
+    return headers;
+}
+function splitGraphemes(text, start, end, maximum) {
+    if (end - start <= maximum)
+        return [end];
+    const ends = [];
+    let cursor = start;
+    let candidate = start;
+    for (const segment of GRAPHEME_SEGMENTER.segment(text.slice(start, end))) {
+        const absoluteEnd = start + segment.index + segment.segment.length;
+        if (absoluteEnd - cursor > maximum) {
+            invariant(candidate > cursor, "document_grapheme_exceeds_atomic_limit");
+            ends.push(candidate);
+            cursor = candidate;
+        }
+        candidate = absoluteEnd;
+    }
+    if (cursor < end)
+        ends.push(end);
+    return ends;
+}
+function sentenceEnds(text, start, end, maximumAtomic) {
+    if (start === end)
+        return [];
+    const normalized = text.slice(start, end).replace(/[\r\n]/gu, " ");
+    const rawEnds = [...SENTENCE_SEGMENTER.segment(normalized)].map((segment) => start + segment.index + segment.segment.length);
+    const result = [];
+    let cursor = start;
+    for (const rawEnd of rawEnds.length > 0 ? rawEnds : [end]) {
+        for (const safeEnd of splitGraphemes(text, cursor, rawEnd, maximumAtomic))
+            result.push(safeEnd);
+        cursor = rawEnd;
+    }
+    if (result.at(-1) !== end)
+        result.push(...splitGraphemes(text, cursor, end, maximumAtomic));
+    return result;
+}
+function buildUnits(text, maximumAtomic) {
+    const headers = chapterHeaders(text);
+    const units = [];
+    let chapterIndex = 0;
+    let chapterId;
+    let cursor = 0;
+    const push = (kind, start, end) => {
+        if (end <= start)
+            return;
+        units.push(Object.freeze({
+            id: `unit-${String(units.length + 1).padStart(7, "0")}`,
+            kind,
+            startOffset: start,
+            endOffset: end,
+            text: text.slice(start, end),
+            ...(chapterId === undefined ? {} : { chapterId }),
+        }));
+    };
+    if (text.charCodeAt(0) === 0xfeff) {
+        if (headers[0]?.start === 1)
+            chapterId = "chapter-001";
+        push("control", 0, 1);
+        cursor = 1;
+    }
+    const boundaries = [...headers, { start: text.length, end: text.length }];
+    for (const header of boundaries) {
+        if (cursor < header.start) {
+            let bodyCursor = cursor;
+            BLANK_SEPARATOR.lastIndex = cursor;
+            let match;
+            while ((match = BLANK_SEPARATOR.exec(text)) !== null && match.index < header.start) {
+                const separatorEnd = Math.min(match.index + match[0].length, header.start);
+                for (const end of sentenceEnds(text, bodyCursor, match.index, maximumAtomic)) {
+                    push("sentence", bodyCursor, end);
+                    bodyCursor = end;
+                }
+                push("layout", match.index, separatorEnd);
+                bodyCursor = separatorEnd;
+                if (separatorEnd < match.index + match[0].length)
+                    break;
+            }
+            for (const end of sentenceEnds(text, bodyCursor, header.start, maximumAtomic)) {
+                push("sentence", bodyCursor, end);
+                bodyCursor = end;
+            }
+            cursor = header.start;
+        }
+        if (header.start === text.length)
+            break;
+        chapterIndex += 1;
+        chapterId = `chapter-${String(chapterIndex).padStart(3, "0")}`;
+        push("chapter_header", header.start, header.end);
+        cursor = header.end;
+    }
+    if (cursor < text.length) {
+        for (const end of sentenceEnds(text, cursor, text.length, maximumAtomic)) {
+            push("sentence", cursor, end);
+            cursor = end;
+        }
+    }
+    invariant(units.length > 0 && units[0].startOffset === 0 && units.at(-1).endOffset === text.length, "document_unit_coverage_incomplete");
+    for (let index = 1; index < units.length; index += 1)
+        invariant(units[index - 1].endOffset === units[index].startOffset, "document_unit_coverage_gap");
+    return Object.freeze(units);
+}
+function buildWindows(text, units, options) {
+    const windows = [];
+    let cursor = 0;
+    while (cursor < units.length) {
+        const chapterId = units[cursor].chapterId;
+        let end = cursor;
+        let size = 0;
+        while (end < units.length && units[end].chapterId === chapterId) {
+            const nextSize = units[end].endOffset - units[end].startOffset;
+            if (end > cursor && (size >= options.targetWindowUtf16 || size + nextSize > options.maximumWindowUtf16))
+                break;
+            size += nextSize;
+            end += 1;
+        }
+        invariant(end > cursor, "document_window_no_progress");
+        let contextStart = cursor;
+        if (options.includePreviousSentenceContext && cursor > 0 && units[cursor - 1].chapterId === chapterId) {
+            contextStart = cursor - 1;
+            while (contextStart > 0 && units[contextStart].kind === "layout" && units[contextStart - 1].chapterId === chapterId)
+                contextStart -= 1;
+        }
+        const core = units.slice(cursor, end);
+        const context = units.slice(contextStart, cursor);
+        windows.push(Object.freeze({
+            id: `window-${String(windows.length + 1).padStart(6, "0")}`,
+            coreUnitIds: Object.freeze(core.map((unit) => unit.id)),
+            contextUnitIds: Object.freeze(context.map((unit) => unit.id)),
+            ...(chapterId === undefined ? {} : { chapterId }),
+            startOffset: units[contextStart].startOffset,
+            coreStartOffset: core[0].startOffset,
+            coreEndOffset: core.at(-1).endOffset,
+            endOffset: core.at(-1).endOffset,
+            text: text.slice(units[contextStart].startOffset, core.at(-1).endOffset),
+        }));
+        cursor = end;
+    }
+    invariant(windows.flatMap((window) => window.coreUnitIds).join("\0") === units.map((unit) => unit.id).join("\0"), "document_window_core_coverage_invalid");
+    return Object.freeze(windows);
+}
+/** 深 Module 的唯一构建 Interface：fatal decode、无损校验、结构切分与窗口化一次完成。 */
+export function buildDocumentSource(input) {
+    invariant(input.bytes instanceof Uint8Array && input.bytes.byteLength > 0, "document_bytes_required");
+    const sourceId = identifier(input.sourceId, "document_source_id_invalid");
+    const sourceRevision = identifier(input.sourceRevision, "document_source_revision_invalid");
+    const scopeId = identifier(input.scopeId, "document_scope_id_invalid");
+    let text;
+    try {
+        text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(input.bytes);
+    }
+    catch {
+        throw new Error("document_utf8_invalid");
+    }
+    invariant(equalBytes(new TextEncoder().encode(text), input.bytes), "document_utf8_roundtrip_mismatch");
+    const targetWindowUtf16 = input.options?.targetWindowUtf16 ?? 6_000;
+    const maximumWindowUtf16 = input.options?.maximumWindowUtf16 ?? 8_000;
+    const maximumAtomicUtf16 = input.options?.maximumAtomicUtf16 ?? 2_000;
+    invariant(Number.isSafeInteger(targetWindowUtf16) && targetWindowUtf16 >= 4_000 && targetWindowUtf16 <= 8_000, "document_target_window_invalid");
+    invariant(Number.isSafeInteger(maximumWindowUtf16) && maximumWindowUtf16 >= targetWindowUtf16 && maximumWindowUtf16 <= 12_000, "document_maximum_window_invalid");
+    invariant(Number.isSafeInteger(maximumAtomicUtf16) && maximumAtomicUtf16 >= 256 && maximumAtomicUtf16 <= targetWindowUtf16, "document_atomic_limit_invalid");
+    const units = buildUnits(text, maximumAtomicUtf16);
+    const windows = buildWindows(text, units, { targetWindowUtf16, maximumWindowUtf16, maximumAtomicUtf16, includePreviousSentenceContext: input.options?.includePreviousSentenceContext ?? true });
+    const sourceSha256 = sha256(input.bytes);
+    return Object.freeze({
+        schemaVersion: DOCUMENT_SOURCE_SCHEMA_VERSION,
+        documentId: sha256(new TextEncoder().encode(JSON.stringify([scopeId, sourceId, sourceRevision, sourceSha256]))),
+        sourceId, sourceRevision, scopeId, sourceSha256,
+        byteLength: input.bytes.byteLength, utf16Length: text.length,
+        originalBytesBase64: Buffer.from(input.bytes).toString("base64"),
+        utf8Bom: input.bytes[0] === 0xef && input.bytes[1] === 0xbb && input.bytes[2] === 0xbf ? "present" : "absent",
+        offsetDomain: "decoded_utf16_including_bom",
+        units, windows,
+    });
+}
+export function documentSourceBytes(document) {
+    const bytes = Uint8Array.from(Buffer.from(document.originalBytesBase64, "base64"));
+    invariant(bytes.byteLength === document.byteLength && sha256(bytes) === document.sourceSha256, "document_original_bytes_invalid");
+    const text = document.units.map((unit) => unit.text).join("");
+    invariant(text.length === document.utf16Length && equalBytes(new TextEncoder().encode(text), bytes), "document_unit_roundtrip_mismatch");
+    return bytes;
+}
+export function validateDocumentPlan(document, plan) {
+    invariant(plan.schemaVersion === DOCUMENT_PLAN_SCHEMA_VERSION, "document_plan_schema_invalid");
+    invariant(plan.documentId === document.documentId, "document_plan_document_mismatch");
+    invariant(Array.isArray(plan.segments) && plan.segments.length > 0, "document_plan_segments_required");
+    const unitById = new Map(document.units.map((unit) => [unit.id, unit]));
+    let expectedStartIndex = 0;
+    const segments = plan.segments.map((segment, segmentIndex) => {
+        const title = identifier(segment.title, "document_plan_segment_title_invalid");
+        const startUnit = unitById.get(segment.startUnitId);
+        const endUnit = unitById.get(segment.endUnitId);
+        invariant(startUnit !== undefined && endUnit !== undefined, "document_plan_segment_unit_unknown");
+        const startIndex = document.units.indexOf(startUnit);
+        const endIndex = document.units.indexOf(endUnit);
+        invariant(startIndex === expectedStartIndex, "document_plan_unit_gap_or_overlap");
+        invariant(endIndex >= startIndex, "document_plan_segment_reverse");
+        const covered = document.units.slice(startIndex, endIndex + 1);
+        invariant(new Set(covered.map((unit) => unit.chapterId ?? null)).size === 1, "document_plan_segment_crosses_chapter");
+        expectedStartIndex = endIndex + 1;
+        return Object.freeze({
+            id: `source-chunk-${String(segmentIndex + 1).padStart(6, "0")}`,
+            startUnitId: startUnit.id,
+            endUnitId: endUnit.id,
+            title,
+            ...(startUnit.chapterId === undefined ? {} : { chapterId: startUnit.chapterId }),
+            startOffset: startUnit.startOffset,
+            endOffset: endUnit.endOffset,
+            indexText: covered.map((unit) => unit.text).join(""),
+        });
+    });
+    invariant(expectedStartIndex === document.units.length, "document_plan_unit_coverage_incomplete");
+    return Object.freeze({ schemaVersion: DOCUMENT_PLAN_SCHEMA_VERSION, documentId: plan.documentId, segments: Object.freeze(segments) });
+}

package/dist/embedding-cache.d.ts

@@ -0,0 +1,35 @@
+import { type AuthoringEmbeddingPort } from "./authoring-model-connection.js";
+export type EmbeddingCacheErrorCode = "invalid_identity" | "invalid_input" | "corrupt" | "provider_response_invalid" | "aborted";
+export declare class EmbeddingCacheError extends Error {
+    readonly code: EmbeddingCacheErrorCode;
+    readonly name = "EmbeddingCacheError";
+    constructor(code: EmbeddingCacheErrorCode);
+}
+export interface EmbeddingCacheCounts {
+    readonly hits: number;
+    readonly misses: number;
+    readonly remoteTexts: number;
+    readonly writes: number;
+}
+export interface CachedEmbeddingPort extends AuthoringEmbeddingPort {
+    readonly origin: string;
+    getCounts(): EmbeddingCacheCounts;
+}
+export type CachedEmbeddingReadOptions = Readonly<{
+    projectRoot: string;
+    embedding: AuthoringEmbeddingPort & Readonly<{
+        origin?: string;
+    }>;
+    origin?: string;
+    inputKind?: "document" | "query";
+}>;
+/** 只读查询现有 corpus cache；任一 miss 返回 null，绝不调用 provider 或创建目录。 */
+export declare function readCachedEmbeddingVectors(options: CachedEmbeddingReadOptions, textSha256s: readonly string[]): Promise<readonly (readonly number[])[] | null>;
+export declare function createCachedEmbeddingPort(options: Readonly<{
+    projectRoot: string;
+    embedding: AuthoringEmbeddingPort & Readonly<{
+        origin?: string;
+    }>;
+    /** 旧 port 尚未暴露 origin 时由 capability 显式注入；不得从 providerId/modelId 推断。 */
+    origin?: string;
+}>): Promise<CachedEmbeddingPort>;