@world-engines/project-host 0.1.0-alpha.0

package/dist/ingestion-source-reconciliation.d.ts

@@ -0,0 +1,88 @@
+import type { LadybugExtractionBatch } from "@world-engines/tmw/natural-language-ingestion";
+type JsonRecord = Record<string, unknown>;
+export type WorldApplyOperation = Readonly<{
+    type: "upsert_entity";
+    id: string;
+    kind: string;
+    props: Readonly<JsonRecord>;
+}> | Readonly<{
+    type: "delete_entity";
+    id: string;
+}> | Readonly<{
+    type: "upsert_relation";
+    src: string;
+    dst: string;
+    kind: string;
+    props: Readonly<JsonRecord>;
+}> | Readonly<{
+    type: "delete_relation";
+    src: string;
+    dst: string;
+    kind: string;
+}>;
+export interface ReconciliationEntity {
+    readonly id: string;
+    readonly kind: string;
+    readonly props: Readonly<JsonRecord>;
+}
+export interface ReconciliationRelation {
+    readonly src: string;
+    readonly dst: string;
+    readonly kind: string;
+    readonly props: Readonly<JsonRecord>;
+}
+export interface IngestionSourceSnapshot {
+    readonly entities: readonly ReconciliationEntity[];
+    readonly relations: readonly ReconciliationRelation[];
+}
+type RelationIdentity = Readonly<{
+    src: string;
+    dst: string;
+    kind: string;
+}>;
+type SeenRevision = Readonly<{
+    source_revision: string;
+    content_sha256: string;
+    batch_id: string;
+}>;
+export interface IngestionSourceMarker {
+    readonly id: string;
+    readonly currentBatchId: string;
+    readonly currentSourceRevision: string;
+    readonly currentContentSha256: string;
+    readonly factIds: readonly string[];
+    readonly entityIds: readonly string[];
+    readonly relations: readonly RelationIdentity[];
+    readonly seenRevisions: readonly SeenRevision[];
+}
+export type IngestionSourceReconciliation = Readonly<{
+    status: "already_committed";
+    marker: IngestionSourceMarker;
+    operations: readonly WorldApplyOperation[];
+}> | Readonly<{
+    status: "apply";
+    marker: IngestionSourceMarker;
+    operations: readonly WorldApplyOperation[];
+}>;
+export declare const INGESTION_SOURCE_CONTROL_KIND = "world_ingestion_source_control";
+declare function factEntityId(scopeId: string, factId: string): string;
+declare function markerId(scopeId: string, sourceId: string): string;
+export declare const ingestionSourceMarkerId: typeof markerId;
+export declare const ingestionFactEntityId: typeof factEntityId;
+export declare function readIngestionSourceMarker(snapshot: IngestionSourceSnapshot, scopeId: string, sourceId: string): IngestionSourceMarker | null;
+export declare function ingestionSourceReconciliationReadIds(input: Readonly<{
+    scopeId: string;
+    sourceId: string;
+    batch: LadybugExtractionBatch;
+    marker: IngestionSourceMarker | null;
+}>): readonly string[];
+export declare function reconcileIngestionSource(input: Readonly<{
+    scopeId: string;
+    sourceId: string;
+    sourceRevision: string;
+    batch: LadybugExtractionBatch;
+    /** 抽取开始前读取到的 source marker batch；world_apply 仍须使用同一 snapshot 的 project revision CAS。 */
+    expectedCurrentBatchId: string | null;
+    snapshot: IngestionSourceSnapshot;
+}>): IngestionSourceReconciliation;
+export {};

package/dist/ingestion-source-reconciliation.js

@@ -0,0 +1,253 @@
+const MARKER_SCHEMA = "world-ingestion-source-marker.v1";
+const MARKER_KIND = "world_ingestion_source_control";
+export const INGESTION_SOURCE_CONTROL_KIND = MARKER_KIND;
+const MANAGED_ENTITY_PROPS = new Set(["scopeId", "sourceEntityId", "label", "facts", "index_text", "embedding", "ingestion_sources"]);
+const MANAGED_RELATION_PROPS = new Set(["scopeId", "factId", "qualifiers", "confidence", "provenance"]);
+function record(value, code) {
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        throw new Error(code);
+    return value;
+}
+function strings(value, code) {
+    if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string"))
+        throw new Error(code);
+    return Object.freeze([...new Set(value)].sort());
+}
+function relationIdentity(value) {
+    const relation = record(value, "authoring_ingestion_source_marker_invalid");
+    if (typeof relation.src !== "string" || typeof relation.dst !== "string" || typeof relation.kind !== "string") {
+        throw new Error("authoring_ingestion_source_marker_invalid");
+    }
+    return Object.freeze({ src: relation.src, dst: relation.dst, kind: relation.kind });
+}
+function relationKey(value) {
+    return JSON.stringify([value.src, value.dst, value.kind]);
+}
+function sortedRelations(values) {
+    return Object.freeze([...new Map(values.map((value) => [relationKey(value), value])).values()]
+        .sort((left, right) => relationKey(left).localeCompare(relationKey(right))));
+}
+function factEntityId(scopeId, factId) {
+    return `${scopeId}:world-fact:${factId}`;
+}
+function markerId(scopeId, sourceId) {
+    return `world-ingestion-source:${encodeURIComponent(scopeId)}:${encodeURIComponent(sourceId)}`;
+}
+export const ingestionSourceMarkerId = markerId;
+export const ingestionFactEntityId = factEntityId;
+function factFrom(value) {
+    const fact = record(value, "authoring_ingestion_fact_invalid");
+    if (typeof fact.factId !== "string" || typeof fact.scopeId !== "string" || !Array.isArray(fact.provenance)) {
+        throw new Error("authoring_ingestion_fact_invalid");
+    }
+    return fact;
+}
+function factsFromBatch(batch, scopeId, sourceId) {
+    const facts = new Map();
+    for (const entity of batch.entities) {
+        const values = entity.props.facts;
+        if (!Array.isArray(values))
+            continue;
+        for (const value of values) {
+            const fact = factFrom(value);
+            if (fact.scopeId !== scopeId || !fact.provenance.some((entry) => entry.sourceId === sourceId)) {
+                throw new Error("authoring_ingestion_source_fact_mismatch");
+            }
+            facts.set(fact.factId, fact);
+        }
+    }
+    return Object.freeze([...facts.values()].sort((left, right) => left.factId.localeCompare(right.factId)));
+}
+function uniqueProvenance(facts) {
+    const values = facts.flatMap((fact) => fact.provenance);
+    return Object.freeze([...new Map(values.map((value) => [JSON.stringify(value), value])).values()]
+        .sort((left, right) => left.sourceId.localeCompare(right.sourceId)
+        || left.startOffset - right.startOffset || left.endOffset - right.endOffset));
+}
+function withoutSource(fact, sourceId) {
+    const provenance = fact.provenance.filter((entry) => entry.sourceId !== sourceId);
+    return provenance.length === 0 ? null : Object.freeze({ ...fact, provenance: Object.freeze(provenance) });
+}
+function mergeFact(existing, incoming) {
+    if (existing === null)
+        return incoming;
+    return Object.freeze({ ...incoming, confidence: Math.max(existing.confidence, incoming.confidence), provenance: uniqueProvenance([existing, incoming]) });
+}
+function renderedFact(fact) {
+    const object = fact.object.type === "entity" ? fact.object.value.label
+        : fact.object.type === "json" ? JSON.stringify(fact.object.value) : String(fact.object.value);
+    return `${fact.subject.label}\n${fact.predicate}: ${object}`;
+}
+function markerFromEntity(entity, scopeId, sourceId) {
+    if (entity === undefined)
+        return null;
+    if (entity.kind !== MARKER_KIND)
+        throw new Error("authoring_ingestion_source_marker_conflict");
+    const props = entity.props;
+    if (props.schema !== MARKER_SCHEMA || props.scope_id !== scopeId || props.source_id !== sourceId
+        || typeof props.current_batch_id !== "string" || typeof props.current_source_revision !== "string"
+        || typeof props.current_content_sha256 !== "string" || !Array.isArray(props.relations)
+        || !Array.isArray(props.seen_revisions))
+        throw new Error("authoring_ingestion_source_marker_invalid");
+    const seenRevisions = props.seen_revisions.map((value) => {
+        const seen = record(value, "authoring_ingestion_source_marker_invalid");
+        if (typeof seen.source_revision !== "string" || typeof seen.content_sha256 !== "string" || typeof seen.batch_id !== "string") {
+            throw new Error("authoring_ingestion_source_marker_invalid");
+        }
+        return Object.freeze({ source_revision: seen.source_revision, content_sha256: seen.content_sha256, batch_id: seen.batch_id });
+    });
+    return Object.freeze({
+        id: entity.id,
+        currentBatchId: props.current_batch_id,
+        currentSourceRevision: props.current_source_revision,
+        currentContentSha256: props.current_content_sha256,
+        factIds: strings(props.fact_ids, "authoring_ingestion_source_marker_invalid"),
+        entityIds: strings(props.entity_ids, "authoring_ingestion_source_marker_invalid"),
+        relations: sortedRelations(props.relations.map(relationIdentity)),
+        seenRevisions: Object.freeze(seenRevisions),
+    });
+}
+export function readIngestionSourceMarker(snapshot, scopeId, sourceId) {
+    const id = markerId(scopeId, sourceId);
+    return markerFromEntity(snapshot.entities.find((entity) => entity.id === id), scopeId, sourceId);
+}
+export function ingestionSourceReconciliationReadIds(input) {
+    const facts = factsFromBatch(input.batch, input.scopeId, input.sourceId);
+    return Object.freeze([...new Set([
+            markerId(input.scopeId, input.sourceId),
+            ...(input.marker?.entityIds ?? []),
+            ...(input.marker?.factIds.map((factId) => factEntityId(input.scopeId, factId)) ?? []),
+            ...input.batch.entities.map((entity) => entity.id),
+            ...facts.map((fact) => factEntityId(input.scopeId, fact.factId)),
+        ])].sort());
+}
+function hasUnmanagedProps(props, managed) {
+    return Object.keys(props).some((key) => !managed.has(key));
+}
+function mergeEntityFacts(existing, incoming, sourceId) {
+    const merged = new Map();
+    const previous = existing?.props.facts;
+    if (Array.isArray(previous))
+        for (const value of previous) {
+            const retained = withoutSource(factFrom(value), sourceId);
+            if (retained !== null)
+                merged.set(retained.factId, retained);
+        }
+    const next = incoming?.props.facts;
+    if (Array.isArray(next))
+        for (const value of next) {
+            const fact = factFrom(value);
+            merged.set(fact.factId, mergeFact(merged.get(fact.factId) ?? null, fact));
+        }
+    return Object.freeze([...merged.values()].sort((left, right) => left.factId.localeCompare(right.factId)));
+}
+function markerEntity(marker, scopeId, sourceId) {
+    return Object.freeze({
+        type: "upsert_entity",
+        id: marker.id,
+        kind: MARKER_KIND,
+        props: Object.freeze({
+            schema: MARKER_SCHEMA,
+            scope_id: scopeId,
+            source_id: sourceId,
+            current_batch_id: marker.currentBatchId,
+            current_source_revision: marker.currentSourceRevision,
+            current_content_sha256: marker.currentContentSha256,
+            fact_ids: marker.factIds,
+            entity_ids: marker.entityIds,
+            relations: marker.relations,
+            seen_revisions: marker.seenRevisions,
+        }),
+    });
+}
+export function reconcileIngestionSource(input) {
+    if (input.batch.source.sourceId !== input.sourceId)
+        throw new Error("authoring_ingestion_source_batch_mismatch");
+    const current = readIngestionSourceMarker(input.snapshot, input.scopeId, input.sourceId);
+    if (current?.currentBatchId === input.batch.batchId)
+        return Object.freeze({ status: "already_committed", marker: current, operations: Object.freeze([]) });
+    if (current?.seenRevisions.some((seen) => seen.batch_id === input.batch.batchId))
+        throw new Error("authoring_ingestion_source_revision_stale");
+    const seenRevision = current?.seenRevisions.find((seen) => seen.source_revision === input.sourceRevision);
+    if (seenRevision !== undefined) {
+        if (seenRevision.content_sha256 !== input.batch.source.contentSha256)
+            throw new Error("authoring_ingestion_source_revision_conflict");
+        throw new Error("authoring_ingestion_source_revision_stale");
+    }
+    if ((current?.currentBatchId ?? null) !== input.expectedCurrentBatchId)
+        throw new Error("authoring_ingestion_source_snapshot_stale");
+    const incomingFacts = factsFromBatch(input.batch, input.scopeId, input.sourceId);
+    const incomingFactIds = incomingFacts.map((fact) => fact.factId);
+    const incomingEntityIds = input.batch.entities.map((entity) => entity.id).sort();
+    const incomingRelations = sortedRelations(input.batch.relations);
+    const entityById = new Map(input.snapshot.entities.map((entity) => [entity.id, entity]));
+    const incomingEntityById = new Map(input.batch.entities.map((entity) => [entity.id, entity]));
+    const incomingFactById = new Map(incomingFacts.map((fact) => [fact.factId, fact]));
+    const operations = [];
+    for (const factId of [...new Set([...(current?.factIds ?? []), ...incomingFactIds])].sort()) {
+        const id = factEntityId(input.scopeId, factId);
+        const existing = entityById.get(id);
+        const existingFact = existing === undefined ? null : withoutSource(factFrom(existing.props.fact), input.sourceId);
+        const incoming = incomingFactById.get(factId);
+        const merged = incoming === undefined ? existingFact : mergeFact(existingFact, incoming);
+        if (merged === null)
+            operations.push(Object.freeze({ type: "delete_entity", id }));
+        else
+            operations.push(Object.freeze({ type: "upsert_entity", id, kind: "world_fact", props: Object.freeze({ ...(existing?.props ?? {}), scope_id: input.scopeId, fact_id: factId, fact: merged, index_text: renderedFact(merged) }) }));
+    }
+    for (const id of [...new Set([...(current?.entityIds ?? []), ...incomingEntityIds])].sort()) {
+        const existing = entityById.get(id);
+        const incoming = incomingEntityById.get(id);
+        const facts = mergeEntityFacts(existing, incoming, input.sourceId);
+        const owners = new Set(strings(existing?.props.ingestion_sources ?? [], "authoring_ingestion_entity_sources_invalid"));
+        owners.delete(input.sourceId);
+        if (incoming !== undefined)
+            owners.add(input.sourceId);
+        if (incoming === undefined && facts.length === 0 && owners.size === 0 && existing !== undefined && !hasUnmanagedProps(existing.props, MANAGED_ENTITY_PROPS)) {
+            operations.push(Object.freeze({ type: "delete_entity", id }));
+            continue;
+        }
+        if (incoming === undefined && existing === undefined)
+            continue;
+        const base = existing?.props ?? {};
+        const incomingProps = incoming?.props ?? {};
+        operations.push(Object.freeze({
+            type: "upsert_entity", id, kind: incoming?.kind ?? existing.kind,
+            props: Object.freeze({ ...base, ...incomingProps, facts, ingestion_sources: Object.freeze([...owners].sort()),
+                ...(incoming === undefined ? {} : { index_text: incoming.text, ...(incoming.embedding === undefined ? {} : { embedding: incoming.embedding }) }) }),
+        }));
+    }
+    const relationByKey = new Map(input.snapshot.relations.map((relation) => [relationKey(relation), relation]));
+    const incomingRelationByKey = new Map(input.batch.relations.map((relation) => [relationKey(relation), relation]));
+    for (const identity of sortedRelations([...(current?.relations ?? []), ...incomingRelations])) {
+        const existing = relationByKey.get(relationKey(identity));
+        const incoming = incomingRelationByKey.get(relationKey(identity));
+        const existingProvenance = Array.isArray(existing?.props.provenance) ? existing.props.provenance.map((value) => record(value, "authoring_ingestion_relation_provenance_invalid")) : [];
+        const retained = existingProvenance.filter((value) => value.sourceId !== input.sourceId);
+        const next = Array.isArray(incoming?.props.provenance) ? incoming.props.provenance.map((value) => record(value, "authoring_ingestion_relation_provenance_invalid")) : [];
+        const provenance = [...new Map([...retained, ...next].map((value) => [JSON.stringify(value), value])).values()];
+        if (incoming === undefined && provenance.length === 0 && existing !== undefined && !hasUnmanagedProps(existing.props, MANAGED_RELATION_PROPS)) {
+            operations.unshift(Object.freeze({ type: "delete_relation", ...identity }));
+        }
+        else if (incoming !== undefined || existing !== undefined) {
+            operations.push(Object.freeze({ type: "upsert_relation", ...identity, props: Object.freeze({ ...(existing?.props ?? {}), ...(incoming?.props ?? {}), provenance: Object.freeze(provenance) }) }));
+        }
+    }
+    const seenRevisions = Object.freeze([...(current?.seenRevisions ?? []), Object.freeze({
+            source_revision: input.sourceRevision,
+            content_sha256: input.batch.source.contentSha256,
+            batch_id: input.batch.batchId,
+        })]);
+    const marker = Object.freeze({
+        id: markerId(input.scopeId, input.sourceId),
+        currentBatchId: input.batch.batchId,
+        currentSourceRevision: input.sourceRevision,
+        currentContentSha256: input.batch.source.contentSha256,
+        factIds: Object.freeze(incomingFactIds),
+        entityIds: Object.freeze(incomingEntityIds),
+        relations: incomingRelations,
+        seenRevisions,
+    });
+    operations.push(markerEntity(marker, input.scopeId, input.sourceId));
+    return Object.freeze({ status: "apply", marker, operations: Object.freeze(operations) });
+}

package/dist/ladybug-runtime.d.ts

@@ -0,0 +1,117 @@
+import type { LadybugExtractionBatch, LadybugSourceProvenance } from "@world-engines/protocol-ts";
+export type { LadybugExtractedEntity, LadybugExtractedRelation, LadybugExtractionBatch, LadybugSourceProvenance, } from "@world-engines/protocol-ts";
+type JsonObject = Readonly<Record<string, unknown>>;
+/**
+ * embedding 是独立 provider capability。实现可以调用本地或远端真实模型，
+ * 但不得在失败时返回 hash/random/零向量等替代品。
+ */
+export interface LadybugEmbeddingPort {
+    readonly providerId: string;
+    readonly modelId: string;
+    readonly dimensions: number;
+    embed(texts: readonly string[], options?: Readonly<{
+        signal?: AbortSignal;
+        inputKind?: "document" | "query";
+    }>): Promise<readonly (readonly number[])[]>;
+}
+export type LadybugRecallMode = "keyword" | "semantic" | "hybrid";
+export type LadybugRecallRequest = Readonly<{
+    queryText: string;
+    mode: LadybugRecallMode;
+    scopeId?: string;
+    topK?: number;
+    /** 可由调用方显式给出；未给出时只能由配置的真实 provider 生成。 */
+    queryEmbedding?: readonly number[];
+    /** cosine distance 上限；超过即视为无关项，不以 topK 强行补齐。 */
+    maxVectorDistance?: number;
+}>;
+export type LadybugRecallHit = Readonly<{
+    id: string;
+    kind: string;
+    scopeId: string | null;
+    text: string;
+    props: JsonObject;
+    score: number;
+    channels: readonly ("keyword" | "semantic")[];
+    sources: readonly LadybugSourceProvenance[];
+}>;
+export type LadybugTraverseRequest = Readonly<{
+    startIds: readonly string[];
+    scopeId?: string;
+    direction?: "out" | "in" | "both";
+    maxHops?: number;
+    relationKinds?: readonly string[];
+    maxPaths?: number;
+    nodeBudget?: number;
+    semanticOnly?: boolean;
+}>;
+export type LadybugTraverseEdge = Readonly<{
+    src: string;
+    dst: string;
+    kind: string;
+    props: JsonObject;
+    sources: readonly LadybugSourceProvenance[];
+}>;
+export type LadybugTraversePath = Readonly<{
+    nodeIds: readonly string[];
+    edges: readonly LadybugTraverseEdge[];
+}>;
+export type LadybugTraverseResult = Readonly<{
+    paths: readonly LadybugTraversePath[];
+    visitedNodeIds: readonly string[];
+    truncated: boolean;
+}>;
+export type LadybugGraphSnapshotRequest = Readonly<{
+    scopeId?: string;
+}>;
+export type LadybugGraphSnapshotNode = Readonly<{
+    id: string;
+    kind: string;
+    scopeId: string | null;
+    props: JsonObject;
+}>;
+export type LadybugGraphSnapshotEdge = Readonly<{
+    src: string;
+    dst: string;
+    kind: string;
+    props: JsonObject;
+}>;
+export type LadybugGraphSnapshot = Readonly<{
+    nodes: readonly LadybugGraphSnapshotNode[];
+    edges: readonly LadybugGraphSnapshotEdge[];
+    nodeCount: number;
+    edgeCount: number;
+}>;
+export type LadybugIngestReceipt = Readonly<{
+    batchId: string;
+    entityCount: number;
+    relationCount: number;
+    statePath: string;
+}>;
+export declare const LADYBUG_WORLD_FACT_QUERY_PROFILE_ID: "world-fact-query/zh-v1";
+export declare const DEFAULT_LADYBUG_MAX_VECTOR_DISTANCE = 0.5;
+/** 仅编码一条查询；corpus 向量由显式准备流程负责，不由该 seam 补齐。 */
+export declare function embedLadybugQuery(port: LadybugEmbeddingPort, text: string): Promise<readonly number[]>;
+export declare function normalizeLadybugGraphSnapshotProps(props: Readonly<Record<string, unknown>>): Record<string, unknown>;
+/** 兼容并行接线中的旧名称；唯一实现仍由 normalizeLadybugGraphSnapshotProps 持有。 */
+export declare const ladybugGraphSnapshotProps: typeof normalizeLadybugGraphSnapshotProps;
+/**
+ * LocalProjectHost 可消费的独立 Ladybug runtime。
+ * JSON 文件是可重建 projection checkpoint；每次 open 都把它装入真实 LadybugDB 并创建 FTS/HNSW 索引。
+ */
+export declare class LadybugProjectRuntime {
+    #private;
+    private constructor();
+    static open(options: {
+        directory: string;
+        dimensions: number;
+        embeddingPort?: LadybugEmbeddingPort;
+        /** shared 仅供已有进程级 Ladybug owner 使用；默认由最后一个 runtime.close 终止 worker。 */
+        workerLifecycle?: "owned" | "shared";
+    }): Promise<LadybugProjectRuntime>;
+    close(): Promise<void>;
+    ingest(batch: LadybugExtractionBatch): Promise<LadybugIngestReceipt>;
+    recall(request: LadybugRecallRequest): Promise<LadybugRecallHit[]>;
+    traverse(request: LadybugTraverseRequest): Promise<LadybugTraverseResult>;
+    readGraphSnapshot(request?: LadybugGraphSnapshotRequest): Promise<LadybugGraphSnapshot>;
+}