kibi-cli 0.17.0 → 0.19.0

package/dist/commands/sync.js

@@ -16,26 +16,103 @@
  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 import { execSync } from "node:child_process";
-import { existsSync } from "node:fs";
+import { existsSync, readdirSync } from "node:fs";
 import * as path from "node:path";
 import { branchErrorToDiagnostic, createDocsNotIndexedDiagnostic, createInvalidAuthoringDiagnostic, createKbMissingDiagnostic, formatSyncSummary, } from "../diagnostics.js";
+import { EngineClient, engineSocketPath, ensureJournaledBranchStoreAsync, fsyncJournaledBranchStore, } from "../engine.js";
 import { isCliDebugEnabled } from "../env.js";
-import { extractFromRelationshipShards, flattenRelationships, validateRelationships, } from "../extractors/relationships.js";
+import { extractRelationshipShard, validateRelationships, } from "../extractors/relationships.js";
+import { analyzeSemanticAdvisorInput } from "../operations/semantic-advisor/analyze-prose.js";
+import { validateSemanticInventoryBoundary } from "../operations/semantic-advisor/ingestion-boundary.js";
 import { PrologProcess } from "../prolog.js";
 import { resolveActiveBranch, } from "../utils/branch-resolver.js";
 import { loadSyncConfig } from "../utils/config.js";
-import { SYNC_CACHE_TTL_MS, SYNC_CACHE_VERSION, hashFile, readSyncCache, toCacheKey, writeSyncCache, } from "./sync/cache.js";
+import { SYNC_CACHE_TTL_MS, SYNC_CACHE_VERSION, hashFile, hashNormalized, readSyncCache, toCacheKey, writeSyncCache, } from "./sync/cache.js";
 import { discoverSourceFiles, } from "./sync/discovery.js";
 import { processExtractions } from "./sync/extraction.js";
 import { refreshManifestCoordinates } from "./sync/manifest.js";
-import { persistEntities, persistRelationships } from "./sync/persistence.js";
-import { atomicPublish, cleanupStaging, createUniqueStagingPath, prepareStagingEnvironment, } from "./sync/staging.js";
+import { persistEntities, persistRelationships, retractEntitiesById, retractEntitiesForSources, retractEntityRelationshipsById, retractRelationships, } from "./sync/persistence.js";
+import { atomicPublish, atomicPublishGeneration, cleanupStaging, createUniqueStagingPath, prepareStagingEnvironment, } from "./sync/staging.js";
 export class SyncError extends Error {
     constructor(message) {
         super(message);
         this.name = "SyncError";
     }
 }
+function relationshipKey(relationship) {
+    return `${relationship.type}\0${relationship.from}\0${relationship.to}`;
+}
+function relationshipFromKey(key) {
+    const [type, from, to] = key.split("\0");
+    if (type === undefined || from === undefined || to === undefined) {
+        throw new SyncError(`Invalid cached relationship key: ${key}`);
+    }
+    return { type, from, to };
+}
+function normalizedEntityHash(result) {
+    const entity = { ...result.entity };
+    // Manifest defaults are generated at extraction time. They must not turn a
+    // one-record edit into 10,000 false-positive symbol updates.
+    entity.created_at = undefined;
+    entity.updated_at = undefined;
+    return hashNormalized({
+        entity,
+        sourceFile: result.sourceFile,
+        relationships: result.relationships
+            .map(relationshipKey)
+            .sort((left, right) => left.localeCompare(right)),
+    });
+}
+async function checkpointNoopSync(workspaceRoot, branch) {
+    const engine = new EngineClient({
+        workspaceRoot,
+        branch,
+        timeout: 120_000,
+    });
+    try {
+        await engine.start();
+        const checkpoint = await engine.checkpoint();
+        if (!checkpoint.success) {
+            throw new SyncError(`Failed to publish the no-op sync checkpoint: ${checkpoint.error ?? "Unknown error"}`);
+        }
+    }
+    finally {
+        await engine.terminate();
+    }
+}
+function compilerCacheHasEntityDelta(cache) {
+    return (cache.entityHashes !== undefined && cache.sourceEntityIds !== undefined);
+}
+function compilerCacheIsFresh(cache, sourceFiles, nowMs) {
+    const sourceKeys = sourceFiles.map(toCacheKey);
+    if (Object.keys(cache.hashes).some((key) => !sourceKeys.includes(key))) {
+        return false;
+    }
+    return sourceKeys.every((key) => {
+        const lastSeen = cache.seenAt[key];
+        const lastSeenMs = lastSeen ? Date.parse(lastSeen) : Number.NaN;
+        return (typeof cache.hashes[key] === "string" &&
+            lastSeen !== undefined &&
+            !Number.isNaN(lastSeenMs) &&
+            nowMs - lastSeenMs <= SYNC_CACHE_TTL_MS);
+    });
+}
+function compilerCacheFilesMatch(cache, sourceFiles, relationshipFiles) {
+    for (const file of sourceFiles) {
+        if (cache.hashes[toCacheKey(file)] !== hashFile(file))
+            return false;
+    }
+    const relationshipKeys = new Set(relationshipFiles.map(toCacheKey));
+    if (Object.keys(cache.relationshipHashes ?? {}).some((key) => !relationshipKeys.has(key))) {
+        return false;
+    }
+    for (const file of relationshipFiles) {
+        if (cache.relationshipHashes?.[toCacheKey(file)] !== hashFile(file)) {
+            return false;
+        }
+    }
+    return true;
+}
 // implements REQ-003, REQ-007
 export async function syncCommand(options = {}, runtime = {}) {
     const validateOnly = options.validateOnly ?? false;
@@ -70,6 +147,24 @@ export async function syncCommand(options = {}, runtime = {}) {
             throw new SyncError(`Failed to resolve active branch: ${branchResult.error}`);
         }
         currentBranch = branchResult.branch;
+        // Cut over legacy branches before creating a staging copy.  If another
+        // CLI/MCP operation has an engine attached, stop that single writer so the
+        // atomic publish below cannot race its RDF lock.
+        const livePathForEngine = path.join(process.cwd(), `.kb/branches/${currentBranch}`);
+        if (!validateOnly) {
+            await ensureJournaledBranchStoreAsync(livePathForEngine);
+            const existingEngine = new EngineClient({
+                workspaceRoot: process.cwd(),
+                branch: currentBranch,
+                timeout: 2_000,
+            });
+            const needsExclusiveGenerationPublish = rebuild || runtime.createProlog !== undefined;
+            if (needsExclusiveGenerationPublish &&
+                existsSync(engineSocketPath(process.cwd(), currentBranch))) {
+                await existingEngine.stop(false).catch(() => undefined);
+                await existingEngine.terminate();
+            }
+        }
         if (isCliDebugEnabled()) {
             // eslint-disable-next-line no-console
             console.log("[kibi-debug] currentBranch:", currentBranch);
@@ -88,16 +183,120 @@ export async function syncCommand(options = {}, runtime = {}) {
         const syncCache = readSyncCache(cachePath);
         const nowIso = new Date().toISOString();
         const nowMs = Date.now();
+        const currentSourceKeys = new Set(sourceFiles.map(toCacheKey));
+        const relationshipShardFiles = existsSync(relationshipsDir)
+            ? readdirSync(relationshipsDir)
+                .filter((file) => file.endsWith(".yaml") || file.endsWith(".yml"))
+                .map((file) => path.join(relationshipsDir, file))
+                .sort()
+            : [];
+        // No-op is a first-class compiler result. Hashing the configured inputs is
+        // sufficient when normalized entity/shard inventories are already fresh;
+        // avoid rebuilding maps and rediscovering RDF state for the common path.
+        if (!validateOnly &&
+            !rebuild &&
+            !options.refreshSymbolCoordinates &&
+            runtime.createProlog === undefined &&
+            compilerCacheHasEntityDelta(syncCache) &&
+            compilerCacheIsFresh(syncCache, sourceFiles, nowMs) &&
+            compilerCacheFilesMatch(syncCache, sourceFiles, relationshipShardFiles)) {
+            await checkpointNoopSync(process.cwd(), currentBranch);
+            console.log("✓ Imported 0 entities, 0 relationships (no changes)");
+            return withOptionalCommit({
+                branch: currentBranch,
+                timestamp: new Date().toISOString(),
+                entityCounts,
+                relationshipCount: 0,
+                success: true,
+                published: false,
+                failures: diagnostics,
+                durationMs: Date.now() - startTime,
+                exitCode: 0,
+            }, getCurrentCommit());
+        }
+        const changedSourceHashes = new Map();
+        for (const file of sourceFiles) {
+            const key = toCacheKey(file);
+            const hash = hashFile(file);
+            if (syncCache.hashes[key] !== hash)
+                changedSourceHashes.set(key, hash);
+        }
         const nextHashes = {};
         const nextSeenAt = {};
-        const shardResults = extractFromRelationshipShards(relationshipsDir);
-        const allRelationships = flattenRelationships(shardResults);
+        const nextSemanticHashes = {
+            ...syncCache.semanticHashes,
+        };
+        const nextSemanticContracts = {
+            ...syncCache.semanticContracts,
+        };
+        const nextRelationshipHashes = {};
+        const nextShardRelationships = {};
+        const addedShardRelationships = [];
+        const removedShardRelationships = [];
+        const initialSemanticBaseline = Object.keys(syncCache.hashes).length === 0;
         const changedMarkdownFiles = [];
         const changedManifestFiles = [];
+        const forceEntitySourceKeys = new Set();
+        // A removed source has no file left to extract, but its entities must be
+        // retracted from the branch. Keep this separate from failed extraction:
+        // malformed current files remain authoritative until a later successful
+        // sync rather than deleting their last known-good entities.
+        const deletedSourceFiles = Object.keys(syncCache.hashes)
+            .filter((key) => !currentSourceKeys.has(key))
+            .map((key) => path.resolve(process.cwd(), key));
+        const currentRelationshipKeys = new Set();
+        for (const shardPath of relationshipShardFiles) {
+            const key = toCacheKey(shardPath);
+            currentRelationshipKeys.add(key);
+            let hash;
+            try {
+                hash = hashFile(shardPath);
+                nextRelationshipHashes[key] = hash;
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                console.warn(`Warning: Failed to hash relationship shard ${shardPath}: ${message}`);
+            }
+            const previousKeys = syncCache.shardRelationships?.[key] ?? [];
+            const shardChanged = syncCache.shardRelationships === undefined ||
+                hash === undefined ||
+                syncCache.relationshipHashes?.[key] !== hash;
+            const currentKeys = shardChanged
+                ? extractRelationshipShard(shardPath)
+                    .relationships.map(relationshipKey)
+                    .sort((left, right) => left.localeCompare(right))
+                : previousKeys;
+            nextShardRelationships[key] = currentKeys;
+            if (shardChanged) {
+                const previous = new Set(previousKeys);
+                const current = new Set(currentKeys);
+                for (const edge of previous) {
+                    if (!current.has(edge)) {
+                        removedShardRelationships.push(relationshipFromKey(edge));
+                    }
+                }
+                for (const edge of current) {
+                    if (!previous.has(edge)) {
+                        addedShardRelationships.push(relationshipFromKey(edge));
+                    }
+                }
+            }
+        }
+        for (const [key, previousKeys] of Object.entries(syncCache.shardRelationships ?? {})) {
+            if (currentRelationshipKeys.has(key))
+                continue;
+            for (const edge of previousKeys) {
+                removedShardRelationships.push(relationshipFromKey(edge));
+            }
+        }
+        const relationshipChanged = addedShardRelationships.length > 0 ||
+            removedShardRelationships.length > 0;
         for (const file of sourceFiles) {
             try {
                 const key = toCacheKey(file);
-                const hash = hashFile(file);
+                const hash = changedSourceHashes.get(key) ?? syncCache.hashes[key];
+                if (hash === undefined)
+                    continue;
                 const lastSeen = syncCache.seenAt[key];
                 const lastSeenMs = lastSeen ? Date.parse(lastSeen) : Number.NaN;
                 const expired = !lastSeen ||
@@ -105,8 +304,10 @@ export async function syncCommand(options = {}, runtime = {}) {
                     nowMs - lastSeenMs > SYNC_CACHE_TTL_MS;
                 nextHashes[key] = hash;
                 nextSeenAt[key] = nowIso;
-                const isChanged = expired || syncCache.hashes[key] !== hash || validateOnly || rebuild;
+                const isChanged = expired || changedSourceHashes.has(key) || validateOnly || rebuild;
                 if (isChanged) {
+                    if (expired)
+                        forceEntitySourceKeys.add(key);
                     if (markdownFiles.includes(file)) {
                         changedMarkdownFiles.push(file);
                     }
@@ -120,9 +321,134 @@ export async function syncCommand(options = {}, runtime = {}) {
                 console.warn(`Warning: Failed to hash ${file}: ${message}`);
             }
         }
+        // A v1 cache has only whole-file hashes. Perform one compiler-metadata
+        // backfill so subsequent edits can be reduced to normalized entity deltas.
+        if (syncCache.entityHashes === undefined ||
+            syncCache.sourceEntityIds === undefined) {
+            for (const file of markdownFiles) {
+                if (!changedMarkdownFiles.includes(file))
+                    changedMarkdownFiles.push(file);
+            }
+            for (const file of manifestFiles) {
+                if (!changedManifestFiles.includes(file))
+                    changedManifestFiles.push(file);
+            }
+        }
+        // Coordinate refresh must precede extraction so the manifest overlay that
+        // is persisted into RDF observes the newly generated coordinates. Force
+        // each refreshed manifest through extraction even when its authored YAML
+        // hash is unchanged; the generated coordinate artifact is an input too.
+        if (!validateOnly && options.refreshSymbolCoordinates) {
+            for (const file of manifestFiles) {
+                try {
+                    await refreshManifestCoordinates(file, process.cwd(), {
+                        refreshSymbolCoordinates: options.refreshSymbolCoordinates,
+                    });
+                    if (!changedManifestFiles.includes(file)) {
+                        changedManifestFiles.push(file);
+                    }
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    console.warn(`Warning: Failed to refresh symbol coordinates in ${file}: ${message}`);
+                }
+            }
+        }
         const performedFullReindex = changedMarkdownFiles.length === markdownFiles.length &&
             changedManifestFiles.length === manifestFiles.length;
-        const { results, failedCacheKeys, errors } = await processExtractions(changedMarkdownFiles, changedManifestFiles, validateOnly);
+        const extraction = await processExtractions(changedMarkdownFiles, changedManifestFiles, validateOnly);
+        const { failedCacheKeys, errors } = extraction;
+        const extractedResults = extraction.results;
+        const nextEntityHashes = {
+            ...(syncCache.entityHashes ?? {}),
+        };
+        const nextSourceEntityIds = {
+            ...(syncCache.sourceEntityIds ?? {}),
+        };
+        const removedEntityIds = new Set();
+        const changedEntityIds = new Set();
+        const supportsEntityDelta = syncCache.entityHashes !== undefined &&
+            syncCache.sourceEntityIds !== undefined;
+        const useEntityDelta = supportsEntityDelta &&
+            !validateOnly &&
+            !rebuild &&
+            runtime.createProlog === undefined;
+        const materializeAllRelationships = !useEntityDelta || changedMarkdownFiles.length > 0;
+        const allRelationships = materializeAllRelationships
+            ? Object.values(nextShardRelationships).flat().map(relationshipFromKey)
+            : [...addedShardRelationships, ...removedShardRelationships];
+        for (const deletedSource of deletedSourceFiles) {
+            const sourceKey = toCacheKey(deletedSource);
+            for (const id of nextSourceEntityIds[sourceKey] ?? []) {
+                removedEntityIds.add(id);
+                delete nextEntityHashes[id];
+            }
+            delete nextSourceEntityIds[sourceKey];
+        }
+        const resultsBySource = new Map();
+        for (const result of extractedResults) {
+            const sourceKey = toCacheKey(result.entity.source);
+            const sourceResults = resultsBySource.get(sourceKey) ?? [];
+            sourceResults.push(result);
+            resultsBySource.set(sourceKey, sourceResults);
+        }
+        for (const changedSource of [
+            ...changedMarkdownFiles,
+            ...changedManifestFiles,
+        ]) {
+            const sourceKey = toCacheKey(changedSource);
+            if (failedCacheKeys.has(sourceKey))
+                continue;
+            const sourceResults = resultsBySource.get(sourceKey) ?? [];
+            const previousIds = new Set(nextSourceEntityIds[sourceKey] ?? []);
+            const currentIds = sourceResults.map(({ entity }) => entity.id).sort();
+            const currentIdSet = new Set(currentIds);
+            for (const previousId of previousIds) {
+                if (!currentIdSet.has(previousId)) {
+                    removedEntityIds.add(previousId);
+                    delete nextEntityHashes[previousId];
+                }
+            }
+            for (const result of sourceResults) {
+                const hash = normalizedEntityHash(result);
+                if (forceEntitySourceKeys.has(sourceKey) ||
+                    nextEntityHashes[result.entity.id] !== hash) {
+                    changedEntityIds.add(result.entity.id);
+                }
+                nextEntityHashes[result.entity.id] = hash;
+            }
+            nextSourceEntityIds[sourceKey] = currentIds;
+        }
+        const results = useEntityDelta
+            ? extractedResults.filter(({ entity }) => changedEntityIds.has(entity.id))
+            : extractedResults;
+        for (const result of extractedResults) {
+            if (result.entity.type !== "req")
+                continue;
+            const key = toCacheKey(result.entity.source);
+            const payload = {
+                type: result.entity.type,
+                id: result.entity.id,
+                properties: result.entity,
+                relationships: result.relationships,
+            };
+            const semantic = analyzeSemanticAdvisorInput({ payload });
+            const boundary = validateSemanticInventoryBoundary(payload, result.relationships, semantic.receipt);
+            const previousSemanticHash = syncCache.semanticHashes[key];
+            const advertisedContract = result.entity.semantic_inventory_version !== undefined ||
+                result.entity.semantic_source_hash !== undefined ||
+                result.entity.semantic_inventory !== undefined;
+            const enforceBoundary = (!initialSemanticBaseline && syncCache.hashes[key] === undefined) ||
+                syncCache.semanticContracts[key] === true ||
+                (previousSemanticHash !== undefined &&
+                    previousSemanticHash !== boundary.sourceHash) ||
+                advertisedContract;
+            if (enforceBoundary && boundary.errors.length > 0) {
+                throw new SyncError(`${key}: proposition-complete ingestion failed: ${boundary.errors.join("; ")}. Run kb_semantic_advisor with the complete requirement prose and preserve its inventory contract.`);
+            }
+            nextSemanticHashes[key] = boundary.sourceHash;
+            nextSemanticContracts[key] = advertisedContract;
+        }
         // Collect INVALID_AUTHORING diagnostics
         for (const err of errors) {
             const error = new Error(err.message);
@@ -139,21 +465,6 @@ export async function syncCommand(options = {}, runtime = {}) {
                 diagnostics.push(createInvalidAuthoringDiagnostic(err.file, embeddedTypes));
             }
         }
-        if (!validateOnly) {
-            if (options.refreshSymbolCoordinates) {
-                for (const file of manifestFiles) {
-                    try {
-                        await refreshManifestCoordinates(file, process.cwd(), {
-                            refreshSymbolCoordinates: options.refreshSymbolCoordinates,
-                        });
-                    }
-                    catch (error) {
-                        const message = error instanceof Error ? error.message : String(error);
-                        console.warn(`Warning: Failed to refresh symbol coordinates in ${file}: ${message}`);
-                    }
-                }
-            }
-        }
         if (validateOnly && errors.length > 0 && results.length === 0) {
             for (const err of errors) {
                 console.error(`${err.file}: ${err.message}`);
@@ -171,21 +482,42 @@ export async function syncCommand(options = {}, runtime = {}) {
                 exitCode: 1,
             }, getCurrentCommit());
         }
-        if (results.length === 0 && allRelationships.length === 0 && !rebuild) {
+        if (results.length === 0 &&
+            removedEntityIds.size === 0 &&
+            !relationshipChanged &&
+            (useEntityDelta || changedMarkdownFiles.length === 0) &&
+            (useEntityDelta || changedManifestFiles.length === 0) &&
+            deletedSourceFiles.length === 0 &&
+            !rebuild) {
             const evictedHashes = {};
             const evictedSeenAt = {};
+            const evictedSemanticHashes = {};
+            const evictedSemanticContracts = {};
             for (const [key, hash] of Object.entries(nextHashes)) {
                 if (failedCacheKeys.has(key)) {
                     continue;
                 }
                 evictedHashes[key] = hash;
                 evictedSeenAt[key] = nextSeenAt[key] ?? nowIso;
+                if (nextSemanticHashes[key] !== undefined) {
+                    evictedSemanticHashes[key] = nextSemanticHashes[key];
+                    evictedSemanticContracts[key] = nextSemanticContracts[key] === true;
+                }
             }
             writeSyncCache(cachePath, {
                 version: SYNC_CACHE_VERSION,
                 hashes: evictedHashes,
+                relationshipHashes: nextRelationshipHashes,
+                entityHashes: nextEntityHashes,
+                sourceEntityIds: nextSourceEntityIds,
+                shardRelationships: nextShardRelationships,
                 seenAt: evictedSeenAt,
+                semanticHashes: evictedSemanticHashes,
+                semanticContracts: evictedSemanticContracts,
             });
+            if (runtime.createProlog === undefined) {
+                await checkpointNoopSync(process.cwd(), currentBranch);
+            }
             console.log("✓ Imported 0 entities, 0 relationships (no changes)");
             return withOptionalCommit({
                 branch: currentBranch,
@@ -204,6 +536,135 @@ export async function syncCommand(options = {}, runtime = {}) {
         if (!kbExists && !rebuild) {
             diagnostics.push(createKbMissingDiagnostic(currentBranch, livePath));
         }
+        // implements REQ-core-journaled-engine-delta-sync
+        // Normal syncs are compiled directly into the long-lived single-writer
+        // engine. Rebuilds retain generation replacement semantics, and injected
+        // Prolog runtimes keep the staging path used by the contract fixtures.
+        // This is the cutover that avoids copying and rewriting a complete branch
+        // for a one-symbol or relationship-only change.
+        if (!validateOnly && !rebuild && runtime.createProlog === undefined) {
+            const engine = new EngineClient({
+                workspaceRoot: process.cwd(),
+                branch: currentBranch,
+                timeout: 120_000,
+            });
+            const engineProlog = engine;
+            try {
+                await engine.start();
+                const successfulChangedSources = [
+                    ...changedMarkdownFiles,
+                    ...changedManifestFiles,
+                ].filter((file) => !failedCacheKeys.has(toCacheKey(file)));
+                const removedCount = useEntityDelta
+                    ? await retractEntitiesById(engineProlog, [...removedEntityIds])
+                    : await retractEntitiesForSources(engineProlog, [
+                        ...successfulChangedSources,
+                        ...deletedSourceFiles,
+                    ]);
+                if (useEntityDelta) {
+                    await retractEntityRelationshipsById(engineProlog, [
+                        ...changedEntityIds,
+                    ]);
+                    const exactRemovedEdges = removedShardRelationships.filter((relationship) => !removedEntityIds.has(relationship.from) &&
+                        !removedEntityIds.has(relationship.to) &&
+                        !changedEntityIds.has(relationship.from));
+                    await retractRelationships(engineProlog, exactRemovedEdges);
+                }
+                else if (relationshipChanged) {
+                    const cleared = await engine.query("kb_retract_all_relationships");
+                    if (!cleared.success) {
+                        throw new SyncError(`Failed to clear changed relationship shards: ${cleared.error || "Unknown error"}`);
+                    }
+                }
+                const entityIds = new Set();
+                for (const { entity } of results) {
+                    entityCounts[entity.type] = (entityCounts[entity.type] || 0) + 1;
+                }
+                const { entityCount, kbModified: entitiesModified } = await persistEntities(engineProlog, results, entityIds, {
+                    loadExistingEntityIds: !useEntityDelta,
+                });
+                const validationErrors = useEntityDelta
+                    ? []
+                    : validateRelationships(allRelationships, entityIds);
+                if (validationErrors.length > 0) {
+                    console.warn(`Warning: ${validationErrors.length} dangling relationship(s) found`);
+                }
+                const danglingKeys = new Set(validationErrors.map(({ relationship }) => `${relationship.type}|${relationship.from}|${relationship.to}`));
+                const shardDeltaByKey = new Map();
+                const shardCandidates = useEntityDelta
+                    ? [
+                        ...addedShardRelationships,
+                        ...allRelationships.filter((relationship) => changedEntityIds.has(relationship.from)),
+                    ]
+                    : allRelationships;
+                for (const relationship of shardCandidates) {
+                    if (!danglingKeys.has(`${relationship.type}|${relationship.from}|${relationship.to}`)) {
+                        shardDeltaByKey.set(relationshipKey(relationship), relationship);
+                    }
+                }
+                const { relationshipCount, kbModified: relationshipsModified } = await persistRelationships(engineProlog, results, [
+                    ...shardDeltaByKey.values(),
+                ]);
+                const kbModified = removedCount > 0 ||
+                    relationshipChanged ||
+                    entitiesModified ||
+                    relationshipsModified;
+                if (kbModified)
+                    engine.invalidateCache();
+                const saveResult = await engine.save();
+                if (!saveResult.success) {
+                    throw new SyncError(`Failed to save journaled KB: ${saveResult.error || "Unknown error"}`);
+                }
+                const evictedHashes = {};
+                const evictedSeenAt = {};
+                const evictedSemanticHashes = {};
+                const evictedSemanticContracts = {};
+                for (const [key, hash] of Object.entries(nextHashes)) {
+                    if (failedCacheKeys.has(key))
+                        continue;
+                    evictedHashes[key] = hash;
+                    evictedSeenAt[key] = nextSeenAt[key] ?? nowIso;
+                    if (nextSemanticHashes[key] !== undefined) {
+                        evictedSemanticHashes[key] = nextSemanticHashes[key];
+                        evictedSemanticContracts[key] = nextSemanticContracts[key] === true;
+                    }
+                }
+                writeSyncCache(path.join(livePath, "sync-cache.json"), {
+                    version: SYNC_CACHE_VERSION,
+                    hashes: evictedHashes,
+                    relationshipHashes: nextRelationshipHashes,
+                    entityHashes: nextEntityHashes,
+                    sourceEntityIds: nextSourceEntityIds,
+                    shardRelationships: nextShardRelationships,
+                    seenAt: evictedSeenAt,
+                    semanticHashes: evictedSemanticHashes,
+                    semanticContracts: evictedSemanticContracts,
+                });
+                published = true;
+                if (performedFullReindex &&
+                    markdownFiles.length > 0 &&
+                    entityCount < markdownFiles.length) {
+                    diagnostics.push(createDocsNotIndexedDiagnostic(markdownFiles.length, entityCount));
+                }
+                console.log(`✓ Imported ${entityCount} entities, ${relationshipCount} relationships (removed ${removedCount} entities)`);
+                const commit = getCurrentCommit();
+                const summary = withOptionalCommit({
+                    branch: currentBranch,
+                    timestamp: new Date().toISOString(),
+                    entityCounts,
+                    relationshipCount,
+                    success: true,
+                    published,
+                    failures: diagnostics,
+                    durationMs: Date.now() - startTime,
+                }, commit);
+                console.log(formatSyncSummary(summary));
+                return { ...summary, exitCode: 0 };
+            }
+            finally {
+                await engine.terminate();
+            }
+        }
         stagingPath = createUniqueStagingPath(currentBranch, process.cwd());
         const runtimeContext = {
             currentBranch,
@@ -227,6 +688,20 @@ export async function syncCommand(options = {}, runtime = {}) {
             for (const { entity } of results) {
                 entityCounts[entity.type] = (entityCounts[entity.type] || 0) + 1;
             }
+            // A copied generation retains entities from old source contents. Remove
+            // only successfully extracted/deleted sources before applying the new
+            // entities; failed extraction deliberately leaves the previous data.
+            await retractEntitiesForSources(prolog, [
+                ...changedMarkdownFiles,
+                ...changedManifestFiles,
+                ...deletedSourceFiles,
+            ].filter((file) => !failedCacheKeys.has(toCacheKey(file))));
+            if (relationshipChanged) {
+                const clearRelationships = await prolog.query("kb_retract_all_relationships");
+                if (!clearRelationships.success) {
+                    throw new SyncError(`Failed to clear changed relationship shards: ${clearRelationships.error || "Unknown error"}`);
+                }
+            }
             const { entityCount, kbModified: entitiesModified } = await persistEntities(prolog, results, entityIds);
             const validationErrors = validateRelationships(allRelationships, entityIds);
             if (validationErrors.length > 0) {
@@ -284,22 +759,41 @@ export async function syncCommand(options = {}, runtime = {}) {
             }
             await prolog.query("kb_detach");
             await prolog.terminate();
-            atomicPublish(stagingPath, livePath);
+            const journaledLive = existsSync(path.join(livePath, "storage.json"));
+            if (rebuild && journaledLive) {
+                atomicPublishGeneration(stagingPath, livePath);
+            }
+            else {
+                atomicPublish(stagingPath, livePath);
+            }
+            fsyncJournaledBranchStore(livePath);
             cleanupStaging(stagingPath);
             const evictedHashes = {};
             const evictedSeenAt = {};
+            const evictedSemanticHashes = {};
+            const evictedSemanticContracts = {};
             for (const [key, hash] of Object.entries(nextHashes)) {
                 if (failedCacheKeys.has(key)) {
                     continue;
                 }
                 evictedHashes[key] = hash;
                 evictedSeenAt[key] = nextSeenAt[key] ?? nowIso;
+                if (nextSemanticHashes[key] !== undefined) {
+                    evictedSemanticHashes[key] = nextSemanticHashes[key];
+                    evictedSemanticContracts[key] = nextSemanticContracts[key] === true;
+                }
             }
             const liveCachePath = path.join(livePath, "sync-cache.json");
             writeSyncCache(liveCachePath, {
                 version: SYNC_CACHE_VERSION,
                 hashes: evictedHashes,
+                relationshipHashes: nextRelationshipHashes,
+                entityHashes: nextEntityHashes,
+                sourceEntityIds: nextSourceEntityIds,
+                shardRelationships: nextShardRelationships,
                 seenAt: evictedSeenAt,
+                semanticHashes: evictedSemanticHashes,
+                semanticContracts: evictedSemanticContracts,
             });
             published = true;
             if (performedFullReindex &&