kibi-cli 0.18.1 → 1.0.1

package/dist/commands/sync.js

@@ -16,42 +16,241 @@
  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 import { execSync } from "node:child_process";
-import { existsSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, writeFileSync, } from "node:fs";
 import * as path from "node:path";
 import { branchErrorToDiagnostic, createDocsNotIndexedDiagnostic, createInvalidAuthoringDiagnostic, createKbMissingDiagnostic, formatSyncSummary, } from "../diagnostics.js";
+import { EngineClient, acquireEnginePublicationLease, engineSocketPath, ensureJournaledBranchStoreAsync, fsyncJournaledBranchStore, } from "../engine.js";
 import { isCliDebugEnabled } from "../env.js";
-import { extractFromRelationshipShards, flattenRelationships, validateRelationships, } from "../extractors/relationships.js";
+import { extractRelationshipShard, validateRelationships, } from "../extractors/relationships.js";
+import { acquireSymbolCompilerLock, releaseSymbolCompilerLock, } from "../operations/mutation/symbol-compiler-lock.js";
 import { analyzeSemanticAdvisorInput } from "../operations/semantic-advisor/analyze-prose.js";
 import { validateSemanticInventoryBoundary } from "../operations/semantic-advisor/ingestion-boundary.js";
 import { PrologProcess } from "../prolog.js";
-import { resolveActiveBranch, } from "../utils/branch-resolver.js";
-import { loadSyncConfig } from "../utils/config.js";
-import { SYNC_CACHE_TTL_MS, SYNC_CACHE_VERSION, hashFile, readSyncCache, toCacheKey, writeSyncCache, } from "./sync/cache.js";
-import { discoverSourceFiles, } from "./sync/discovery.js";
-import { processExtractions } from "./sync/extraction.js";
+import { resolveBranchAttachment } from "../utils/branch-resolver.js";
+import { branchStoreManifestPath, branchStorePath, ensureBranchStoreManifest, expectedBranchStoreManifest, } from "../utils/branch-store-locator.js";
+import { loadEntityPaths } from "../utils/config.js";
+import { SYNC_CACHE_TTL_MS, SYNC_CACHE_VERSION, hashFile, hashManifestWithCoordinates, hashNormalized, readSyncCache, toCacheKey, writeSyncCache, } from "./sync/cache.js";
+import { clearRecoveredPendingSourceReceipts, discoverSourceFiles, } from "./sync/discovery.js";
+import { normalizeExtractionSources, processExtractions, } from "./sync/extraction.js";
 import { refreshManifestCoordinates } from "./sync/manifest.js";
-import { persistEntities, persistRelationships } from "./sync/persistence.js";
-import { atomicPublish, cleanupStaging, createUniqueStagingPath, prepareStagingEnvironment, } from "./sync/staging.js";
+import { persistEntities, persistRelationships, retractEntitiesById, retractEntitiesForSources, retractEntityRelationshipsById, retractRelationships, } from "./sync/persistence.js";
+import { atomicPublish, atomicPublishGeneration, cleanupStaging, createUniqueStagingPath, prepareStagingEnvironment, } from "./sync/staging.js";
 export class SyncError extends Error {
     constructor(message) {
         super(message);
         this.name = "SyncError";
     }
 }
+function relationshipKey(relationship) {
+    return `${relationship.type}\0${relationship.from}\0${relationship.to}`;
+}
+function relationshipFromKey(key) {
+    const [type, from, to] = key.split("\0");
+    if (type === undefined || from === undefined || to === undefined) {
+        throw new SyncError(`Invalid cached relationship key: ${key}`);
+    }
+    return { type, from, to };
+}
+function assertNoUnresolvedGitConflicts(workspaceRoot) {
+    let conflicted = "";
+    try {
+        conflicted = execSync("git diff --name-only --diff-filter=U --", {
+            cwd: workspaceRoot,
+            encoding: "utf8",
+            timeout: 5000,
+            stdio: ["pipe", "pipe", "pipe"],
+        }).trim();
+    }
+    catch {
+        // A non-Git workspace is handled by branch resolution below. Do not turn
+        // a missing Git binary into a misleading conflict diagnostic.
+        return;
+    }
+    if (conflicted) {
+        throw new SyncError(`Unresolved Git conflicts block Kibi compilation; resolve these authored files first: ${conflicted.split("\n").join(", ")}`);
+    }
+}
+function trackedRelationshipFiles(workspaceRoot, relationshipsDir, recoverMissingPendingSources = false, recoveredPendingReceiptPaths = []) {
+    let tracked;
+    try {
+        tracked = new Set(execSync("git ls-files --cached -- .kb/relationships", {
+            cwd: workspaceRoot,
+            encoding: "utf8",
+            timeout: 5000,
+            stdio: ["pipe", "pipe", "pipe"],
+        })
+            .split("\n")
+            .filter(Boolean)
+            .map((file) => file.replaceAll("\\", "/")));
+    }
+    catch {
+        // A non-Git workspace has no tracked relationship inputs. The caller's
+        // branch resolver will produce the user-facing attachment diagnostic.
+        return [];
+    }
+    const pending = new Set();
+    const pendingRoot = path.join(workspaceRoot, ".kb", "recovery", "pending-sources");
+    if (existsSync(pendingRoot)) {
+        for (const receiptName of readdirSync(pendingRoot)) {
+            if (!receiptName.endsWith(".json"))
+                continue;
+            let receipt;
+            try {
+                receipt = JSON.parse(readFileSync(path.join(pendingRoot, receiptName), "utf8"));
+            }
+            catch {
+                // Malformed receipts are left for explicit recovery diagnostics. They
+                // must never make an unrelated relationship shard disappear silently.
+                continue;
+            }
+            if (typeof receipt.path !== "string" ||
+                typeof receipt.afterHash !== "string") {
+                continue;
+            }
+            const relative = receipt.path.replaceAll("\\", "/");
+            if (!relative.startsWith(".kb/relationships/"))
+                continue;
+            const absolute = path.resolve(workspaceRoot, relative);
+            if (!absolute.startsWith(`${path.resolve(workspaceRoot)}${path.sep}`)) {
+                throw new SyncError(`Pending source path escapes workspace: ${relative}`);
+            }
+            if (!existsSync(absolute)) {
+                if (recoverMissingPendingSources) {
+                    const receiptPath = path.join(pendingRoot, receiptName);
+                    if (!recoveredPendingReceiptPaths.some((candidate) => candidate.receiptPath === receiptPath)) {
+                        recoveredPendingReceiptPaths.push({
+                            receiptPath,
+                            path: relative,
+                            afterHash: receipt.afterHash,
+                            rawHash: createHash("sha256")
+                                .update(readFileSync(receiptPath))
+                                .digest("hex"),
+                        });
+                    }
+                    continue;
+                }
+                throw new SyncError(`Pending source is missing: ${relative}`);
+            }
+            const actual = createHash("sha256")
+                .update(readFileSync(absolute))
+                .digest("hex");
+            if (actual !== receipt.afterHash) {
+                throw new SyncError(`Pending source hash drift blocks sync for ${relative}`);
+            }
+            pending.add(relative);
+        }
+    }
+    try {
+        return readdirSync(relationshipsDir)
+            .filter((file) => file.endsWith(".yaml") || file.endsWith(".yml"))
+            .map((file) => path.join(relationshipsDir, file))
+            .filter((file) => {
+            const relative = path
+                .relative(workspaceRoot, file)
+                .replaceAll(path.sep, "/");
+            return tracked.has(relative) || pending.has(relative);
+        })
+            .sort();
+    }
+    catch {
+        return [];
+    }
+}
+function normalizedEntityHash(result) {
+    const entity = { ...result.entity };
+    // Manifest defaults are generated at extraction time. They must not turn a
+    // one-record edit into 10,000 false-positive symbol updates.
+    entity.created_at = undefined;
+    entity.updated_at = undefined;
+    return hashNormalized({
+        entity,
+        sourceFile: result.sourceFile,
+        relationships: result.relationships
+            .map(relationshipKey)
+            .sort((left, right) => left.localeCompare(right)),
+    });
+}
+// implements REQ-kibi-proposition-complete-ingestion
+function semanticBoundaryRelationships(result, shardRelationships) {
+    const relationships = new Map();
+    for (const relationship of [...result.relationships, ...shardRelationships]) {
+        if (relationship.from !== result.entity.id)
+            continue;
+        relationships.set(relationshipKey(relationship), relationship);
+    }
+    return [...relationships.values()];
+}
+async function checkpointNoopSync(workspaceRoot, branch) {
+    const engine = new EngineClient({
+        workspaceRoot,
+        branch,
+        timeout: 120_000,
+    });
+    try {
+        await engine.start();
+        const checkpoint = await engine.checkpoint();
+        if (!checkpoint.success) {
+            throw new SyncError(`Failed to publish the no-op sync checkpoint: ${checkpoint.error ?? "Unknown error"}`);
+        }
+    }
+    finally {
+        await engine.terminate();
+    }
+}
+function compilerCacheHasEntityDelta(cache) {
+    return (cache.entityHashes !== undefined && cache.sourceEntityIds !== undefined);
+}
+function compilerCacheIsFresh(cache, workspaceRoot, sourceFiles, nowMs) {
+    const sourceKeys = sourceFiles.map((file) => toCacheKey(workspaceRoot, file));
+    if (Object.keys(cache.hashes).some((key) => !sourceKeys.includes(key))) {
+        return false;
+    }
+    return sourceKeys.every((key) => {
+        const lastSeen = cache.seenAt[key];
+        const lastSeenMs = lastSeen ? Date.parse(lastSeen) : Number.NaN;
+        return (typeof cache.hashes[key] === "string" &&
+            lastSeen !== undefined &&
+            !Number.isNaN(lastSeenMs) &&
+            nowMs - lastSeenMs <= SYNC_CACHE_TTL_MS);
+    });
+}
+function compilerCacheFilesMatch(cache, workspaceRoot, sourceFiles, relationshipFiles, sourceHashOf) {
+    for (const file of sourceFiles) {
+        if (cache.hashes[toCacheKey(workspaceRoot, file)] !== sourceHashOf(file))
+            return false;
+    }
+    const relationshipKeys = new Set(relationshipFiles.map((file) => toCacheKey(workspaceRoot, file)));
+    if (Object.keys(cache.relationshipHashes ?? {}).some((key) => !relationshipKeys.has(key))) {
+        return false;
+    }
+    for (const file of relationshipFiles) {
+        if (cache.relationshipHashes?.[toCacheKey(workspaceRoot, file)] !==
+            hashFile(workspaceRoot, file)) {
+            return false;
+        }
+    }
+    return true;
+}
 // implements REQ-003, REQ-007
 export async function syncCommand(options = {}, runtime = {}) {
     const validateOnly = options.validateOnly ?? false;
     const rebuild = options.rebuild ?? false;
+    const recoveryBackupPath = options.recoveryBackupPath;
+    const workspaceRoot = path.resolve(options.workspaceRoot ?? process.cwd());
     const startTime = Date.now();
     const diagnostics = [];
     const entityCounts = {};
     let published = false;
     let currentBranch;
     let stagingPath;
+    let publicationLease;
+    let compilerLock;
+    let operationFailure;
     const getCurrentCommit = () => {
         try {
             return execSync("git rev-parse HEAD", {
-                cwd: process.cwd(),
+                cwd: workspaceRoot,
                 encoding: "utf8",
                 timeout: 5000,
                 stdio: ["pipe", "pipe", "pipe"],
@@ -63,22 +262,72 @@ export async function syncCommand(options = {}, runtime = {}) {
     };
     const withOptionalCommit = (value, commit) => commit !== undefined ? { ...value, commit } : value;
     try {
+        if (!validateOnly) {
+            compilerLock = await (runtime.acquireSymbolCompilerLock ?? acquireSymbolCompilerLock)(workspaceRoot);
+        }
+        assertNoUnresolvedGitConflicts(workspaceRoot);
         // Branch resolution
-        const branchResult = resolveActiveBranch(process.cwd());
+        const branchResult = resolveBranchAttachment(workspaceRoot);
         if ("error" in branchResult) {
             const diagnostic = branchErrorToDiagnostic(branchResult.code, branchResult.error);
             diagnostics.push(diagnostic);
             console.error(`Failed to resolve active branch: ${branchResult.error}`);
             throw new SyncError(`Failed to resolve active branch: ${branchResult.error}`);
         }
-        currentBranch = branchResult.branch;
+        if (branchResult.migrationRequired && !validateOnly) {
+            throw new SyncError(`Sync blocked: KB is attached through legacy branch storage for '${branchResult.gitBranch}'. Run 'kibi branch migrate --from ${branchResult.kbBranch} --to ${branchResult.gitBranch} --apply' first.`);
+        }
+        currentBranch = branchResult.kbBranch;
+        // Cut over legacy branches before creating a staging copy.  If another
+        // CLI/MCP operation has an engine attached, stop that single writer so the
+        // atomic publish below cannot race its RDF lock.
+        const livePathForEngine = branchStorePath(workspaceRoot, currentBranch);
+        if (!validateOnly && recoveryBackupPath === undefined) {
+            ensureBranchStoreManifest(workspaceRoot, currentBranch);
+            await ensureJournaledBranchStoreAsync(livePathForEngine);
+        }
+        const needsExclusiveGenerationPublish = !validateOnly && (rebuild || recoveryBackupPath !== undefined);
+        if (needsExclusiveGenerationPublish) {
+            // Recovery intentionally avoids attaching the broken live store. The
+            // clean rebuild is compiled into staging and published only after it
+            // can be reopened successfully.
+            const existingEngine = new EngineClient({
+                workspaceRoot,
+                branch: currentBranch,
+                timeout: 2_000,
+                allowPublicationLock: true,
+            });
+            const engineSocketExists = existsSync(engineSocketPath(workspaceRoot, currentBranch));
+            try {
+                publicationLease = acquireEnginePublicationLease(workspaceRoot, currentBranch);
+                // This client is explicitly allowed to attach under its own lease so
+                // a daemon that appeared between the socket probe and lease creation
+                // is still stopped before publication.
+                if (engineSocketExists ||
+                    existsSync(engineSocketPath(workspaceRoot, currentBranch))) {
+                    await existingEngine.start(false);
+                }
+                if (existingEngine.isRunning()) {
+                    await existingEngine.stop(false);
+                }
+                else if (engineSocketExists ||
+                    existsSync(engineSocketPath(workspaceRoot, currentBranch))) {
+                    throw new SyncError(`Kibi engine socket remains present but is not reachable: ${engineSocketPath(workspaceRoot, currentBranch)}`);
+                }
+            }
+            finally {
+                await existingEngine.terminate();
+            }
+        }
         if (isCliDebugEnabled()) {
             // eslint-disable-next-line no-console
             console.log("[kibi-debug] currentBranch:", currentBranch);
         }
-        const config = loadSyncConfig(process.cwd());
-        const paths = config.paths;
-        const { markdownFiles, manifestFiles, relationshipsDir } = await discoverSourceFiles(process.cwd(), paths);
+        const paths = loadEntityPaths(workspaceRoot);
+        const { markdownFiles, manifestFiles, relationshipsDir, recoveredPendingReceiptPaths, } = await discoverSourceFiles(workspaceRoot, {
+            trackedOnly: true,
+            recoverMissingPendingSources: recoveryBackupPath !== undefined,
+        });
         if (isCliDebugEnabled()) {
             // eslint-disable-next-line no-console
             console.log("[kibi-debug] markdownFiles:", markdownFiles.length);
@@ -86,10 +335,70 @@ export async function syncCommand(options = {}, runtime = {}) {
             console.log("[kibi-debug] manifestFiles:", manifestFiles.length);
         }
         const sourceFiles = [...markdownFiles, ...manifestFiles].sort();
-        const cachePath = path.join(process.cwd(), `.kb/branches/${currentBranch}/sync-cache.json`);
+        const cachePath = path.join(livePathForEngine, "sync-cache.json");
         const syncCache = readSyncCache(cachePath);
         const nowIso = new Date().toISOString();
         const nowMs = Date.now();
+        const currentSourceKeys = new Set(sourceFiles.map((file) => toCacheKey(workspaceRoot, file)));
+        const relationshipShardFiles = existsSync(relationshipsDir)
+            ? trackedRelationshipFiles(workspaceRoot, relationshipsDir, recoveryBackupPath !== undefined, recoveredPendingReceiptPaths)
+            : [];
+        // No-op is a first-class compiler result. Hashing the configured inputs is
+        // sufficient when normalized entity/shard inventories are already fresh;
+        // avoid rebuilding maps and rediscovering RDF state for the common path.
+        // Symbol manifests hash together with their generated coordinate artifact
+        // so artifact-only changes never masquerade as no-ops.
+        const fastPathSourceHashOf = (file) => manifestFiles.includes(file)
+            ? hashManifestWithCoordinates(workspaceRoot, file, path.join(path.dirname(file), "symbol-coordinates.yaml"))
+            : hashFile(workspaceRoot, file);
+        if (!validateOnly &&
+            !rebuild &&
+            !options.refreshSymbolCoordinates &&
+            runtime.createProlog === undefined &&
+            compilerCacheHasEntityDelta(syncCache) &&
+            compilerCacheIsFresh(syncCache, workspaceRoot, sourceFiles, nowMs) &&
+            compilerCacheFilesMatch(syncCache, workspaceRoot, sourceFiles, relationshipShardFiles, fastPathSourceHashOf)) {
+            await checkpointNoopSync(workspaceRoot, currentBranch);
+            console.log("✓ Imported 0 entities, 0 relationships (no changes)");
+            return withOptionalCommit({
+                branch: currentBranch,
+                timestamp: new Date().toISOString(),
+                entityCounts,
+                relationshipCount: 0,
+                success: true,
+                published: false,
+                failures: diagnostics,
+                durationMs: Date.now() - startTime,
+                exitCode: 0,
+            }, getCurrentCommit());
+        }
+        // Explicit coordinate refresh must precede fingerprinting so the manifest
+        // overlay that is persisted into RDF observes the newly generated
+        // coordinates. Refresh failures are fatal: cache state may never advance
+        // on a partially published compiler dependency.
+        const refreshedManifestKeys = new Set();
+        if (!validateOnly && options.refreshSymbolCoordinates) {
+            for (const file of manifestFiles) {
+                await refreshManifestCoordinates(file, workspaceRoot, {
+                    refreshSymbolCoordinates: options.refreshSymbolCoordinates,
+                });
+                refreshedManifestKeys.add(toCacheKey(workspaceRoot, file));
+            }
+        }
+        // The generated coordinate artifact is a compiler dependency of every
+        // symbol manifest: its existence, emptiness, content, and deletion must
+        // all change the effective source hash even when authored YAML is stable.
+        const coordinatesPathForManifest = (manifestFile) => path.join(path.dirname(manifestFile), "symbol-coordinates.yaml");
+        const effectiveSourceHash = (file) => manifestFiles.includes(file)
+            ? hashManifestWithCoordinates(workspaceRoot, file, coordinatesPathForManifest(file))
+            : hashFile(workspaceRoot, file);
+        const changedSourceHashes = new Map();
+        for (const file of sourceFiles) {
+            const key = toCacheKey(workspaceRoot, file);
+            const hash = effectiveSourceHash(file);
+            if (syncCache.hashes[key] !== hash)
+                changedSourceHashes.set(key, hash);
+        }
         const nextHashes = {};
         const nextSeenAt = {};
         const nextSemanticHashes = {
@@ -98,15 +407,74 @@ export async function syncCommand(options = {}, runtime = {}) {
         const nextSemanticContracts = {
             ...syncCache.semanticContracts,
         };
+        const nextRelationshipHashes = {};
+        const nextShardRelationships = {};
+        const addedShardRelationships = [];
+        const removedShardRelationships = [];
         const initialSemanticBaseline = Object.keys(syncCache.hashes).length === 0;
-        const shardResults = extractFromRelationshipShards(relationshipsDir);
-        const allRelationships = flattenRelationships(shardResults);
         const changedMarkdownFiles = [];
         const changedManifestFiles = [];
+        const forceEntitySourceKeys = new Set();
+        // A removed source has no file left to extract, but its entities must be
+        // retracted from the branch. Keep this separate from failed extraction:
+        // malformed current files remain authoritative until a later successful
+        // sync rather than deleting their last known-good entities.
+        const deletedSourceFiles = Object.keys(syncCache.hashes)
+            .filter((key) => !currentSourceKeys.has(key))
+            .map((key) => path.resolve(workspaceRoot, key));
+        const currentRelationshipKeys = new Set();
+        for (const shardPath of relationshipShardFiles) {
+            const key = toCacheKey(workspaceRoot, shardPath);
+            currentRelationshipKeys.add(key);
+            let hash;
+            try {
+                hash = hashFile(workspaceRoot, shardPath);
+                nextRelationshipHashes[key] = hash;
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                console.warn(`Warning: Failed to hash relationship shard ${shardPath}: ${message}`);
+            }
+            const previousKeys = syncCache.shardRelationships?.[key] ?? [];
+            const shardChanged = syncCache.shardRelationships === undefined ||
+                hash === undefined ||
+                syncCache.relationshipHashes?.[key] !== hash;
+            const currentKeys = shardChanged
+                ? extractRelationshipShard(shardPath)
+                    .relationships.map(relationshipKey)
+                    .sort((left, right) => left.localeCompare(right))
+                : previousKeys;
+            nextShardRelationships[key] = currentKeys;
+            if (shardChanged) {
+                const previous = new Set(previousKeys);
+                const current = new Set(currentKeys);
+                for (const edge of previous) {
+                    if (!current.has(edge)) {
+                        removedShardRelationships.push(relationshipFromKey(edge));
+                    }
+                }
+                for (const edge of current) {
+                    if (!previous.has(edge)) {
+                        addedShardRelationships.push(relationshipFromKey(edge));
+                    }
+                }
+            }
+        }
+        for (const [key, previousKeys] of Object.entries(syncCache.shardRelationships ?? {})) {
+            if (currentRelationshipKeys.has(key))
+                continue;
+            for (const edge of previousKeys) {
+                removedShardRelationships.push(relationshipFromKey(edge));
+            }
+        }
+        const relationshipChanged = addedShardRelationships.length > 0 ||
+            removedShardRelationships.length > 0;
         for (const file of sourceFiles) {
             try {
-                const key = toCacheKey(file);
-                const hash = hashFile(file);
+                const key = toCacheKey(workspaceRoot, file);
+                const hash = changedSourceHashes.get(key) ?? syncCache.hashes[key];
+                if (hash === undefined)
+                    continue;
                 const lastSeen = syncCache.seenAt[key];
                 const lastSeenMs = lastSeen ? Date.parse(lastSeen) : Number.NaN;
                 const expired = !lastSeen ||
@@ -114,8 +482,10 @@ export async function syncCommand(options = {}, runtime = {}) {
                     nowMs - lastSeenMs > SYNC_CACHE_TTL_MS;
                 nextHashes[key] = hash;
                 nextSeenAt[key] = nowIso;
-                const isChanged = expired || syncCache.hashes[key] !== hash || validateOnly || rebuild;
+                const isChanged = expired || changedSourceHashes.has(key) || validateOnly || rebuild;
                 if (isChanged) {
+                    if (expired)
+                        forceEntitySourceKeys.add(key);
                     if (markdownFiles.includes(file)) {
                         changedMarkdownFiles.push(file);
                     }
@@ -129,41 +499,121 @@ export async function syncCommand(options = {}, runtime = {}) {
                 console.warn(`Warning: Failed to hash ${file}: ${message}`);
             }
         }
-        // Coordinate refresh must precede extraction so the manifest overlay that
-        // is persisted into RDF observes the newly generated coordinates. Force
-        // each refreshed manifest through extraction even when its authored YAML
-        // hash is unchanged; the generated coordinate artifact is an input too.
-        if (!validateOnly && options.refreshSymbolCoordinates) {
+        // A v1 cache has only whole-file hashes. Perform one compiler-metadata
+        // backfill so subsequent edits can be reduced to normalized entity deltas.
+        if (syncCache.entityHashes === undefined ||
+            syncCache.sourceEntityIds === undefined) {
+            for (const file of markdownFiles) {
+                if (!changedMarkdownFiles.includes(file))
+                    changedMarkdownFiles.push(file);
+            }
             for (const file of manifestFiles) {
-                try {
-                    await refreshManifestCoordinates(file, process.cwd(), {
-                        refreshSymbolCoordinates: options.refreshSymbolCoordinates,
-                    });
-                    if (!changedManifestFiles.includes(file)) {
-                        changedManifestFiles.push(file);
-                    }
-                }
-                catch (error) {
-                    const message = error instanceof Error ? error.message : String(error);
-                    console.warn(`Warning: Failed to refresh symbol coordinates in ${file}: ${message}`);
-                }
+                if (!changedManifestFiles.includes(file))
+                    changedManifestFiles.push(file);
             }
         }
+        // Explicitly requested refreshes force coordinate-bearing symbols through
+        // extraction and RDF persistence even when normalized entity hashes match
+        // the cached state, repairing warm-cache RDF divergence.
+        for (const file of manifestFiles) {
+            if (!refreshedManifestKeys.has(toCacheKey(workspaceRoot, file)) ||
+                changedManifestFiles.includes(file)) {
+                continue;
+            }
+            changedManifestFiles.push(file);
+        }
+        for (const key of refreshedManifestKeys) {
+            forceEntitySourceKeys.add(key);
+        }
         const performedFullReindex = changedMarkdownFiles.length === markdownFiles.length &&
             changedManifestFiles.length === manifestFiles.length;
-        const { results, failedCacheKeys, errors } = await processExtractions(changedMarkdownFiles, changedManifestFiles, validateOnly);
-        for (const result of results) {
+        const extraction = await processExtractions(changedMarkdownFiles, changedManifestFiles, validateOnly, workspaceRoot);
+        const { failedCacheKeys, errors } = extraction;
+        const extractedResults = normalizeExtractionSources(extraction.results, workspaceRoot);
+        const nextEntityHashes = {
+            ...(syncCache.entityHashes ?? {}),
+        };
+        const nextSourceEntityIds = {
+            ...(syncCache.sourceEntityIds ?? {}),
+        };
+        const removedEntityIds = new Set();
+        const changedEntityIds = new Set();
+        const supportsEntityDelta = syncCache.entityHashes !== undefined &&
+            syncCache.sourceEntityIds !== undefined;
+        const useEntityDelta = supportsEntityDelta &&
+            !validateOnly &&
+            !rebuild &&
+            runtime.createProlog === undefined;
+        // Re-materialize every authored shard edge whenever a source entity is
+        // refreshed. Entity-delta sync retracts outgoing relationships for changed
+        // entities before persisting their extracted relationships. Manifest
+        // entities commonly carry no inline links, so a manifest-only edit (for
+        // example, a generated coordinate refresh) would otherwise drop valid
+        // shard-only edges and leave the cache claiming they were already synced.
+        const materializeAllRelationships = !useEntityDelta ||
+            changedMarkdownFiles.length > 0 ||
+            changedManifestFiles.length > 0;
+        const allRelationships = materializeAllRelationships
+            ? Object.values(nextShardRelationships).flat().map(relationshipFromKey)
+            : [...addedShardRelationships, ...removedShardRelationships];
+        for (const deletedSource of deletedSourceFiles) {
+            const sourceKey = toCacheKey(workspaceRoot, deletedSource);
+            for (const id of nextSourceEntityIds[sourceKey] ?? []) {
+                removedEntityIds.add(id);
+                delete nextEntityHashes[id];
+            }
+            delete nextSourceEntityIds[sourceKey];
+        }
+        const resultsBySource = new Map();
+        for (const result of extractedResults) {
+            const sourceKey = toCacheKey(workspaceRoot, result.entity.source);
+            const sourceResults = resultsBySource.get(sourceKey) ?? [];
+            sourceResults.push(result);
+            resultsBySource.set(sourceKey, sourceResults);
+        }
+        for (const changedSource of [
+            ...changedMarkdownFiles,
+            ...changedManifestFiles,
+        ]) {
+            const sourceKey = toCacheKey(workspaceRoot, changedSource);
+            if (failedCacheKeys.has(sourceKey))
+                continue;
+            const sourceResults = resultsBySource.get(sourceKey) ?? [];
+            const previousIds = new Set(nextSourceEntityIds[sourceKey] ?? []);
+            const currentIds = sourceResults.map(({ entity }) => entity.id).sort();
+            const currentIdSet = new Set(currentIds);
+            for (const previousId of previousIds) {
+                if (!currentIdSet.has(previousId)) {
+                    removedEntityIds.add(previousId);
+                    delete nextEntityHashes[previousId];
+                }
+            }
+            for (const result of sourceResults) {
+                const hash = normalizedEntityHash(result);
+                if (forceEntitySourceKeys.has(sourceKey) ||
+                    nextEntityHashes[result.entity.id] !== hash) {
+                    changedEntityIds.add(result.entity.id);
+                }
+                nextEntityHashes[result.entity.id] = hash;
+            }
+            nextSourceEntityIds[sourceKey] = currentIds;
+        }
+        const results = useEntityDelta
+            ? extractedResults.filter(({ entity }) => changedEntityIds.has(entity.id))
+            : extractedResults;
+        for (const result of extractedResults) {
             if (result.entity.type !== "req")
                 continue;
-            const key = toCacheKey(result.entity.source);
+            const key = toCacheKey(workspaceRoot, result.entity.source);
+            const relationships = semanticBoundaryRelationships(result, allRelationships);
             const payload = {
                 type: result.entity.type,
                 id: result.entity.id,
                 properties: result.entity,
-                relationships: result.relationships,
+                relationships,
             };
             const semantic = analyzeSemanticAdvisorInput({ payload });
-            const boundary = validateSemanticInventoryBoundary(payload, result.relationships, semantic.receipt);
+            const boundary = validateSemanticInventoryBoundary(payload, relationships, semantic.receipt);
             const previousSemanticHash = syncCache.semanticHashes[key];
             const advertisedContract = result.entity.semantic_inventory_version !== undefined ||
                 result.entity.semantic_source_hash !== undefined ||
@@ -212,7 +662,13 @@ export async function syncCommand(options = {}, runtime = {}) {
                 exitCode: 1,
             }, getCurrentCommit());
         }
-        if (results.length === 0 && allRelationships.length === 0 && !rebuild) {
+        if (results.length === 0 &&
+            removedEntityIds.size === 0 &&
+            !relationshipChanged &&
+            (useEntityDelta || changedMarkdownFiles.length === 0) &&
+            (useEntityDelta || changedManifestFiles.length === 0) &&
+            deletedSourceFiles.length === 0 &&
+            !rebuild) {
             const evictedHashes = {};
             const evictedSeenAt = {};
             const evictedSemanticHashes = {};
@@ -228,9 +684,19 @@ export async function syncCommand(options = {}, runtime = {}) {
                     evictedSemanticContracts[key] = nextSemanticContracts[key] === true;
                 }
             }
+            // The durable KB checkpoint must precede the cache write: if the
+            // checkpoint fails, cached fingerprints must stay stale so the next
+            // sync recompiles instead of trusting uncommitted compiler output.
+            if (runtime.createProlog === undefined) {
+                await checkpointNoopSync(workspaceRoot, currentBranch);
+            }
             writeSyncCache(cachePath, {
                 version: SYNC_CACHE_VERSION,
                 hashes: evictedHashes,
+                relationshipHashes: nextRelationshipHashes,
+                entityHashes: nextEntityHashes,
+                sourceEntityIds: nextSourceEntityIds,
+                shardRelationships: nextShardRelationships,
                 seenAt: evictedSeenAt,
                 semanticHashes: evictedSemanticHashes,
                 semanticContracts: evictedSemanticContracts,
@@ -248,12 +714,141 @@ export async function syncCommand(options = {}, runtime = {}) {
                 exitCode: 0,
             }, getCurrentCommit());
         }
-        const livePath = path.join(process.cwd(), `.kb/branches/${currentBranch}`);
+        const livePath = branchStorePath(workspaceRoot, currentBranch);
         const kbExists = existsSync(livePath);
         if (!kbExists && !rebuild) {
             diagnostics.push(createKbMissingDiagnostic(currentBranch, livePath));
         }
-        stagingPath = createUniqueStagingPath(currentBranch, process.cwd());
+        // implements REQ-core-journaled-engine-delta-sync
+        // Normal syncs are compiled directly into the long-lived single-writer
+        // engine. Rebuilds retain generation replacement semantics, and injected
+        // Prolog runtimes keep the staging path used by the contract fixtures.
+        // This is the cutover that avoids copying and rewriting a complete branch
+        // for a one-symbol or relationship-only change.
+        if (!validateOnly && !rebuild && runtime.createProlog === undefined) {
+            const engine = new EngineClient({
+                workspaceRoot,
+                branch: currentBranch,
+                timeout: 120_000,
+            });
+            const engineProlog = engine;
+            try {
+                await engine.start();
+                const successfulChangedSources = [
+                    ...changedMarkdownFiles,
+                    ...changedManifestFiles,
+                ].filter((file) => !failedCacheKeys.has(toCacheKey(workspaceRoot, file)));
+                const removedCount = useEntityDelta
+                    ? await retractEntitiesById(engineProlog, [...removedEntityIds])
+                    : await retractEntitiesForSources(engineProlog, [
+                        ...successfulChangedSources,
+                        ...deletedSourceFiles,
+                    ]);
+                if (useEntityDelta) {
+                    await retractEntityRelationshipsById(engineProlog, [
+                        ...changedEntityIds,
+                    ]);
+                    const exactRemovedEdges = removedShardRelationships.filter((relationship) => !removedEntityIds.has(relationship.from) &&
+                        !removedEntityIds.has(relationship.to) &&
+                        !changedEntityIds.has(relationship.from));
+                    await retractRelationships(engineProlog, exactRemovedEdges);
+                }
+                else if (relationshipChanged) {
+                    const cleared = await engine.query("kb_retract_all_relationships");
+                    if (!cleared.success) {
+                        throw new SyncError(`Failed to clear changed relationship shards: ${cleared.error || "Unknown error"}`);
+                    }
+                }
+                const entityIds = new Set();
+                for (const { entity } of results) {
+                    entityCounts[entity.type] = (entityCounts[entity.type] || 0) + 1;
+                }
+                const { entityCount, kbModified: entitiesModified } = await persistEntities(engineProlog, results, entityIds, {
+                    loadExistingEntityIds: !useEntityDelta,
+                });
+                const validationErrors = useEntityDelta
+                    ? []
+                    : validateRelationships(allRelationships, entityIds);
+                if (validationErrors.length > 0) {
+                    console.warn(`Warning: ${validationErrors.length} dangling relationship(s) found`);
+                }
+                const danglingKeys = new Set(validationErrors.map(({ relationship }) => `${relationship.type}|${relationship.from}|${relationship.to}`));
+                const shardDeltaByKey = new Map();
+                const shardCandidates = useEntityDelta && !materializeAllRelationships
+                    ? [
+                        ...addedShardRelationships,
+                        ...allRelationships.filter((relationship) => changedEntityIds.has(relationship.from)),
+                    ]
+                    : allRelationships;
+                for (const relationship of shardCandidates) {
+                    if (!danglingKeys.has(`${relationship.type}|${relationship.from}|${relationship.to}`)) {
+                        shardDeltaByKey.set(relationshipKey(relationship), relationship);
+                    }
+                }
+                const { relationshipCount, kbModified: relationshipsModified } = await persistRelationships(engineProlog, results, [
+                    ...shardDeltaByKey.values(),
+                ]);
+                const kbModified = removedCount > 0 ||
+                    relationshipChanged ||
+                    entitiesModified ||
+                    relationshipsModified;
+                if (kbModified)
+                    engine.invalidateCache();
+                const saveResult = await engine.save();
+                if (!saveResult.success) {
+                    throw new SyncError(`Failed to save journaled KB: ${saveResult.error || "Unknown error"}`);
+                }
+                const evictedHashes = {};
+                const evictedSeenAt = {};
+                const evictedSemanticHashes = {};
+                const evictedSemanticContracts = {};
+                for (const [key, hash] of Object.entries(nextHashes)) {
+                    if (failedCacheKeys.has(key))
+                        continue;
+                    evictedHashes[key] = hash;
+                    evictedSeenAt[key] = nextSeenAt[key] ?? nowIso;
+                    if (nextSemanticHashes[key] !== undefined) {
+                        evictedSemanticHashes[key] = nextSemanticHashes[key];
+                        evictedSemanticContracts[key] = nextSemanticContracts[key] === true;
+                    }
+                }
+                writeSyncCache(path.join(livePath, "sync-cache.json"), {
+                    version: SYNC_CACHE_VERSION,
+                    hashes: evictedHashes,
+                    relationshipHashes: nextRelationshipHashes,
+                    entityHashes: nextEntityHashes,
+                    sourceEntityIds: nextSourceEntityIds,
+                    shardRelationships: nextShardRelationships,
+                    seenAt: evictedSeenAt,
+                    semanticHashes: evictedSemanticHashes,
+                    semanticContracts: evictedSemanticContracts,
+                });
+                published = true;
+                if (performedFullReindex &&
+                    markdownFiles.length > 0 &&
+                    entityCount < markdownFiles.length) {
+                    diagnostics.push(createDocsNotIndexedDiagnostic(markdownFiles.length, entityCount));
+                }
+                console.log(`✓ Imported ${entityCount} entities, ${relationshipCount} relationships (removed ${removedCount} entities)`);
+                const commit = getCurrentCommit();
+                const summary = withOptionalCommit({
+                    branch: currentBranch,
+                    timestamp: new Date().toISOString(),
+                    entityCounts,
+                    relationshipCount,
+                    success: true,
+                    published,
+                    failures: diagnostics,
+                    durationMs: Date.now() - startTime,
+                }, commit);
+                console.log(formatSyncSummary(summary));
+                return { ...summary, exitCode: 0 };
+            }
+            finally {
+                await engine.terminate();
+            }
+        }
+        stagingPath = createUniqueStagingPath(currentBranch, workspaceRoot);
         const runtimeContext = {
             currentBranch,
             livePath,
@@ -262,6 +857,12 @@ export async function syncCommand(options = {}, runtime = {}) {
             validateOnly,
         };
         await prepareStagingEnvironment(stagingPath, livePath, rebuild);
+        // Staging is a new compiled artifact, so install the exact identity fence
+        // before the engine attaches. The manifest is published with the staged
+        // store during the atomic swap.
+        if (!existsSync(branchStoreManifestPath(stagingPath))) {
+            writeFileSync(branchStoreManifestPath(stagingPath), `${JSON.stringify(expectedBranchStoreManifest(currentBranch), null, 2)}\n`, { mode: 0o600 });
+        }
         try {
             const prolog = runtime.createProlog?.({ timeout: 120000 }) ??
                 new PrologProcess({ timeout: 120000 });
@@ -276,6 +877,20 @@ export async function syncCommand(options = {}, runtime = {}) {
             for (const { entity } of results) {
                 entityCounts[entity.type] = (entityCounts[entity.type] || 0) + 1;
             }
+            // A copied generation retains entities from old source contents. Remove
+            // only successfully extracted/deleted sources before applying the new
+            // entities; failed extraction deliberately leaves the previous data.
+            await retractEntitiesForSources(prolog, [
+                ...changedMarkdownFiles,
+                ...changedManifestFiles,
+                ...deletedSourceFiles,
+            ].filter((file) => !failedCacheKeys.has(toCacheKey(workspaceRoot, file))));
+            if (relationshipChanged) {
+                const clearRelationships = await prolog.query("kb_retract_all_relationships");
+                if (!clearRelationships.success) {
+                    throw new SyncError(`Failed to clear changed relationship shards: ${clearRelationships.error || "Unknown error"}`);
+                }
+            }
             const { entityCount, kbModified: entitiesModified } = await persistEntities(prolog, results, entityIds);
             const validationErrors = validateRelationships(allRelationships, entityIds);
             if (validationErrors.length > 0) {
@@ -333,8 +948,46 @@ export async function syncCommand(options = {}, runtime = {}) {
             }
             await prolog.query("kb_detach");
             await prolog.terminate();
-            atomicPublish(stagingPath, livePath);
-            cleanupStaging(stagingPath);
+            const journaledLive = existsSync(path.join(livePath, "storage.json"));
+            if (recoveryBackupPath !== undefined) {
+                if (!existsSync(livePath)) {
+                    throw new SyncError(`Recovery target disappeared before publication: ${livePath}`);
+                }
+                if (existsSync(recoveryBackupPath)) {
+                    throw new SyncError(`Recovery backup path already exists: ${recoveryBackupPath}`);
+                }
+                mkdirSync(path.dirname(recoveryBackupPath), { recursive: true });
+                // Same-filesystem renames give us a recoverable two-step publication:
+                // the original bytes stay at the reported backup path if anything
+                // later needs forensic inspection.
+                renameSync(livePath, recoveryBackupPath);
+                try {
+                    renameSync(stagingPath, livePath);
+                }
+                catch (error) {
+                    renameSync(recoveryBackupPath, livePath);
+                    throw error;
+                }
+            }
+            else if (rebuild && journaledLive) {
+                atomicPublishGeneration(stagingPath, livePath);
+            }
+            else {
+                atomicPublish(stagingPath, livePath);
+            }
+            fsyncJournaledBranchStore(livePath);
+            try {
+                if (recoveryBackupPath !== undefined &&
+                    recoveredPendingReceiptPaths.length > 0) {
+                    clearRecoveredPendingSourceReceipts(workspaceRoot, recoveredPendingReceiptPaths);
+                }
+            }
+            finally {
+                // Receipt compare-and-delete can fail closed after publication when a
+                // newer receipt wins a race.  Staging bytes are still disposable;
+                // leave the published store and surface the pending intent error.
+                cleanupStaging(stagingPath);
+            }
             const evictedHashes = {};
             const evictedSeenAt = {};
             const evictedSemanticHashes = {};
@@ -354,6 +1007,10 @@ export async function syncCommand(options = {}, runtime = {}) {
             writeSyncCache(liveCachePath, {
                 version: SYNC_CACHE_VERSION,
                 hashes: evictedHashes,
+                relationshipHashes: nextRelationshipHashes,
+                entityHashes: nextEntityHashes,
+                sourceEntityIds: nextSourceEntityIds,
+                shardRelationships: nextShardRelationships,
                 seenAt: evictedSeenAt,
                 semanticHashes: evictedSemanticHashes,
                 semanticContracts: evictedSemanticContracts,
@@ -385,6 +1042,7 @@ export async function syncCommand(options = {}, runtime = {}) {
         }
     }
     catch (error) {
+        operationFailure = { error };
         if (stagingPath) {
             cleanupStaging(stagingPath);
         }
@@ -409,5 +1067,9 @@ export async function syncCommand(options = {}, runtime = {}) {
         }
         throw error;
     }
+    finally {
+        publicationLease?.release();
+        releaseSymbolCompilerLock(compilerLock, operationFailure);
+    }
 }
 export { normalizeMarkdownPath } from "./sync/discovery.js";