@neuralis/brain-core 0.1.0 → 0.2.0

package/dist/src/sync/DeltaSyncRunner.js

@@ -1,8 +1,11 @@
+import { syncIndexedFile, createSyncJobEmbedder, isBinaryFile } from '../usecases/syncIndexedFile.js';
+import { looksBinaryContent, BINARY_SNIFF_CHARS } from '../domain/text.js';
 import { createHash } from 'node:crypto';
 import { deriveSourceKind, normalizeSource } from '../domain/source.js';
 import { resolveOsUri } from '../domain/paths.js';
 import { uriAncestors, extractFilename } from '../domain/uri.js';
 import { buildVersionInfo } from '../domain/versioning.js';
+import { embeddingUnavailableReason } from '../embeddings/factory.js';
 import { isEmbeddingBudgetStopError } from '../embeddings/usage.js';
 import { ConnectorNotFoundError } from '@neuralis/package-system/contracts';
 import { compileSyncFilters, isUriIndexableUnderFilters } from '../domain/syncignore.js';
@@ -11,11 +14,11 @@ import { isAppendShaped } from '../domain/chunker.js';
 import { getBrainConfigProvider, getBrainConfigNumber } from '../config/env.js';
 import { resolveSyncCategory, computeContributionEnrichment, buildEnrichmentRefresh, isEnrichmentCurrent, } from './categoryInference.js';
 import { MoveClaims } from './moveDetector.js';
-import { upsertChunksForContent, relocateChunksForParent, pickUriNode } from '../usecases/chunkUpsert.js';
+import { pickUriNode } from '../usecases/chunkUpsert.js';
 import { uriLogHash } from '../domain/contentHash.js';
 import { purgeExcludedPoints } from '../usecases/purgeExcludedPoints.js';
 import { emptyMediaSkipCounts, mediaKindOf, mediaLaneOn, mediaSkipSummary, upsertMediaFromSource, } from '../usecases/mediaLane.js';
-import { syncEmbeddingText, syncStoredText, hashSyncedFile, storedVectorMatchesText, isVectorMissingError, tombstoneSyncedPoint, readHealContent, } from '../usecases/syncPointWrites.js';
+import { hashSyncedFile, tombstoneSyncedPoint, } from '../usecases/syncPointWrites.js';
 import { noopLogger } from '@neuralis/package-system/data';
 const SCROLL_BATCH = 200;
 /**
@@ -46,6 +49,10 @@ const INVENTORY_PAYLOAD_FIELDS = [
     // A payload-only write REPLACES the payload (`overwritePayload`), so the
     // vector's model marker must travel with every row a writer might spread.
     'embed_model',
+    'embedding_status',
+    'embedding_input_hash',
+    'embedding_generation',
+    'embedding_updated_at',
     // …and so must a media point's modality (the purge's lane decision reads it too).
     'modality',
     'recovery_only',
@@ -87,19 +94,6 @@ function limitConcurrency(tasks, concurrency, signal) {
     const workers = Array.from({ length: Math.min(concurrency, tasks.length) }, () => worker());
     return Promise.all(workers).then(() => results);
 }
-const BINARY_EXTENSIONS = new Set([
-    '.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico', '.bmp',
-    '.mp4', '.webm', '.avi', '.mov', '.mkv',
-    '.mp3', '.wav', '.ogg', '.flac', '.aac',
-    '.zip', '.tar', '.gz', '.bz2', '.7z', '.rar',
-    '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
-    '.exe', '.dll', '.so', '.dylib', '.wasm',
-    '.ttf', '.otf', '.woff', '.woff2', '.eot',
-]);
-function isBinaryFile(filename) {
-    const ext = filename.slice(filename.lastIndexOf('.')).toLowerCase();
-    return BINARY_EXTENSIONS.has(ext);
-}
 function isMissingFileError(error) {
     return typeof error === 'object' && error !== null && 'code' in error
         && error.code === 'ENOENT';
@@ -277,7 +271,10 @@ export class DeltaSyncRunner {
                 const uri = typeof point.payload.uri === 'string' ? point.payload.uri : '';
                 if (!uri || !shouldSyncUri(uri) || knownUris.has(uri))
                     continue;
-                if (point.payload.status === 'deleted' || point.payload.category === 'chunk')
+                // A point under review is never paired as a move: relocating it would
+                // strand its review record at the old uri. It is tombstoned below like
+                // any vanished file, and the new uri is an ordinary add.
+                if (point.payload.status === 'deleted' || point.payload.status === 'pending_review' || point.payload.category === 'chunk')
                     continue;
                 disappeared.set(uri, {
                     pointId: point.id,
@@ -292,11 +289,15 @@ export class DeltaSyncRunner {
             let filesMoved = 0;
             let totalChunksEmbedded = 0;
             let totalChunksSkipped = 0;
+            const embeddingReasons = new Set();
             const mediaSkipped = emptyMediaSkipCounts();
             const syncErrors = [];
             const movedOldRefs = new Set();
             emitProgress({ phase: 'processing', processed: 0, total: changed.length });
             const syncConcurrency = Math.max(1, getBrainConfigNumber('brainSyncConcurrency', 'BRAIN_SYNC_CONCURRENCY', DEFAULT_SYNC_CONCURRENCY));
+            const binarySniffChars = binarySniffCharsSetting();
+            const binaryUris = new Set();
+            let binarySkipped = 0;
             let processed = 0;
             // Credential-use deny / embedding daily cap short-circuit: the first
             // typed stop error stops every not-yet-started task — a denied window
@@ -312,7 +313,7 @@ export class DeltaSyncRunner {
                     return { entry, upserted: { action: 'skipped', chunksEmbedded: 0, chunksSkipped: 0 }, error: null };
                 }
                 const existing = existingByUri.get(entry.uri) ?? null;
-                return this.#upsertFromSource(normalizedJob, connector, entry, existing, moves)
+                return this.#upsertFromSource(normalizedJob, connector, entry, existing, moves, binarySniffChars)
                     .then((upserted) => ({ entry, upserted, error: null }))
                     .catch((error) => {
                     if (normalizedJob.signal?.aborted)
@@ -331,6 +332,8 @@ export class DeltaSyncRunner {
                 if (r.status !== 'fulfilled' || !r.value)
                     continue;
                 const { entry, upserted, error } = r.value;
+                if (upserted?.embeddingDegraded)
+                    embeddingReasons.add(upserted.embeddingDegraded);
                 if (error || !upserted) {
                     filesSkipped += 1;
                     if (syncErrors.length < 20)
@@ -340,6 +343,10 @@ export class DeltaSyncRunner {
                     filesSkipped += 1;
                     if (upserted.mediaSkip)
                         mediaSkipped[upserted.mediaSkip] += 1;
+                    if (upserted.binarySkip)
+                        binarySkipped += 1;
+                    if (upserted.binarySkip === 'content')
+                        binaryUris.add(entry.uri);
                 }
                 else {
                     if (upserted.action === 'added')
@@ -419,30 +426,11 @@ export class DeltaSyncRunner {
                     const point = sweepPoints.get(uri);
                     if (!point)
                         continue; // vanished between the inventory and the hydrate
-                    // A content-less point (written before the write path kept inline
-                    // content) is HEALED from disk when the file still holds the bytes
-                    // its vector was built from — the uri already passed the index gate
-                    // above. Otherwise its category is never re-derived from an empty
-                    // string: that would demote a declared contribution the write path
-                    // already resolved.
-                    const inline = typeof point.payload.content === 'string' ? point.payload.content : undefined;
-                    let healed;
-                    try {
-                        healed = inline === undefined ? await readHealContent(connector, point) : undefined;
-                    }
-                    catch (error) {
-                        // A heal is an optional backfill for a point whose vector is fine:
-                        // an unreadable file is reported, never a failed pass — that would
-                        // freeze the checkpoint and park the source in error for good.
-                        this.#logger.warn('sync.runner.heal.read_failed', {
-                            uri,
-                            code: error?.code ?? null,
-                            error: error instanceof Error ? error.message : String(error),
-                        });
-                    }
-                    const content = inline ?? healed;
-                    const derived = buildEnrichmentRefresh(point.payload, uri, content, computeContributionEnrichment(uri, content ?? ''));
-                    const refreshed = healed === undefined ? derived : { ...(derived ?? point.payload), content: healed };
+                    // A point without inline text (a media point) never has its category
+                    // re-derived from an empty string: that would demote a declared
+                    // contribution the write path already resolved.
+                    const content = typeof point.payload.content === 'string' ? point.payload.content : undefined;
+                    const refreshed = buildEnrichmentRefresh(point.payload, uri, content, computeContributionEnrichment(uri, content ?? ''));
                     // Write even when nothing DRIFTED: this is the one-time upgrade pass
                     // that lands the stamp, and without it an already-correct point stays
                     // unstamped and is re-hydrated by every later full sync. The write is
@@ -452,13 +440,12 @@ export class DeltaSyncRunner {
                     // EXCEPT for a point whose content is not in hand: `stampEnrichment`
                     // REMOVES the stamp from an underivable payload by design, so such a
                     // point can never become current and an unconditional write here would
-                    // re-write it on EVERY full pass, forever. A heal that found no
-                    // matching bytes leaves the point here. Drift still writes; a no-op
+                    // re-write it on EVERY full pass, forever. Drift still writes; a no-op
                     // upgrade on an underivable point does not.
                     if (!refreshed && content === undefined)
                         continue;
                     try {
-                        await this.#deps.vector.upsert({ id: point.id, payload: refreshed ?? point.payload });
+                        await this.#deps.vector.writePayload({ id: point.id, payload: refreshed ?? point.payload });
                         if (refreshed)
                             enrichmentRefreshed += 1;
                     }
@@ -497,6 +484,9 @@ export class DeltaSyncRunner {
                     const media = mediaKindOf(extractFilename(String(point.payload.uri ?? '')) ?? '');
                     return media !== null && !mediaLaneOn(normalizedJob.syncConfig.media, media.kind);
                 },
+                // A file this pass READ and judged binary by content: a point an
+                // earlier pass left is withdrawn here, recovery-aware like a pattern.
+                binaryUris,
                 signal: normalizedJob.signal,
             });
             for (const message of purged.errors)
@@ -583,12 +573,14 @@ export class DeltaSyncRunner {
                 chunksEmbedded: totalChunksEmbedded,
                 chunksSkipped: totalChunksSkipped,
                 durationMs: Date.now() - startedAt,
+                ...(embeddingReasons.size ? { embeddingDegraded: [...embeddingReasons] } : {}),
                 ...(filesTotal !== undefined ? { filesTotal } : {}),
                 ...(indexedTotal !== undefined ? { indexedTotal } : {}),
                 ...(unindexedTotal !== undefined ? { unindexedTotal } : {}),
                 ...(fullScan.truncated ? { truncated: true, filesScanCap: scanCap } : {}),
                 ...(checkpoint ? { checkpoint } : {}),
                 ...(mediaSkipSummary(mediaSkipped) ? { mediaSkipped } : {}),
+                ...(binarySkipped > 0 ? { binarySkipped } : {}),
                 ...(syncErrors.length > 0 ? {
                     error: `${syncErrors.length} sync item(s) failed: ${syncErrors.join(' | ')}`,
                     errors: syncErrors,
@@ -668,6 +660,7 @@ export class DeltaSyncRunner {
         let filesMoved = 0;
         let totalChunksEmbedded = 0;
         let totalChunksSkipped = 0;
+        const embeddingReasons = new Set();
         const syncErrors = [];
         const movedOldRefs = new Set();
         const disappeared = new Map();
@@ -677,7 +670,8 @@ export class DeltaSyncRunner {
                 continue;
             disappeared.set(uri, {
                 pointId: point.id,
-                contentHash: typeof point.payload.content_hash === 'string' ? point.payload.content_hash : '',
+                // No hash, no move claim: a point under review is tombstoned, never relocated.
+                contentHash: point.payload.status !== 'pending_review' && typeof point.payload.content_hash === 'string' ? point.payload.content_hash : '',
             });
         }
         const moves = new MoveClaims(disappeared);
@@ -699,6 +693,8 @@ export class DeltaSyncRunner {
         // fail-fasts every tick; documented residual.)
         let useLimitedError = null;
         const mediaSkipped = emptyMediaSkipCounts();
+        const binarySniffChars = binarySniffCharsSetting();
+        let binarySkipped = 0;
         const targetedTasks = changedUris.map((uri) => async () => {
             if (job.signal?.aborted || useLimitedError)
                 return null;
@@ -718,7 +714,7 @@ export class DeltaSyncRunner {
                     modifiedAt: now,
                     ...(typeof stat?.size === 'number' ? { size: stat.size } : {}),
                 };
-                const upserted = await this.#upsertFromSource(job, connector, entry, existing, moves);
+                const upserted = await this.#upsertFromSource(job, connector, entry, existing, moves, binarySniffChars);
                 return { uri, existing, upserted, error: null };
             }
             catch (error) {
@@ -741,6 +737,8 @@ export class DeltaSyncRunner {
                 continue;
             const { uri, existing, upserted, error } = settled.value;
             if (upserted) {
+                if (upserted.embeddingDegraded)
+                    embeddingReasons.add(upserted.embeddingDegraded);
                 if (upserted.action === 'added')
                     filesAdded += 1;
                 else if (upserted.action === 'updated')
@@ -754,6 +752,8 @@ export class DeltaSyncRunner {
                     filesSkipped += 1;
                     if (upserted.mediaSkip)
                         mediaSkipped[upserted.mediaSkip] += 1;
+                    if (upserted.binarySkip)
+                        binarySkipped += 1;
                 }
                 totalChunksEmbedded += upserted.chunksEmbedded;
                 totalChunksSkipped += upserted.chunksSkipped;
@@ -818,7 +818,9 @@ export class DeltaSyncRunner {
             chunksEmbedded: totalChunksEmbedded,
             chunksSkipped: totalChunksSkipped,
             durationMs: Date.now() - startedAt,
+            ...(embeddingReasons.size ? { embeddingDegraded: [...embeddingReasons] } : {}),
             ...(mediaSkipSummary(mediaSkipped) ? { mediaSkipped } : {}),
+            ...(binarySkipped > 0 ? { binarySkipped } : {}),
             ...(cancelled ? { error: 'Sync cancelled' } : syncErrors.length > 0 ? {
                 error: `${syncErrors.length} sync item(s) failed: ${syncErrors.join(' | ')}`,
                 errors: syncErrors,
@@ -945,13 +947,23 @@ export class DeltaSyncRunner {
         } while (cursor);
         return points;
     }
+    #embeddingByJob = new WeakMap();
+    #embedderFor(job) {
+        let embedder = this.#embeddingByJob.get(job);
+        if (!embedder) {
+            embedder = createSyncJobEmbedder(this.#deps.embedder);
+            this.#embeddingByJob.set(job, embedder);
+        }
+        return embedder;
+    }
     /** The ACTIVE model's modalities, asked once per job (a text-only port answers `['text']`). */
     #modalitiesByJob = new WeakMap();
     #modalitiesFor(job) {
         let pending = this.#modalitiesByJob.get(job);
         if (!pending) {
             pending = this.#deps.embedder.modalities
-                ? this.#deps.embedder.modalities().catch(() => ['text'])
+                ? this.#deps.embedder.modalities().catch(error => { if (!embeddingUnavailableReason(error))
+                    throw error; return ['text']; })
                 : Promise.resolve(['text']);
             this.#modalitiesByJob.set(job, pending);
         }
@@ -970,7 +982,7 @@ export class DeltaSyncRunner {
         const id = existing?.id ?? deterministicExternalNodeId(job, uri);
         const now = this.#deps.clock.nowIso();
         const osUri = resolveOsUri(uri, connector);
-        const outcome = await upsertMediaFromSource({ vector: this.#deps.vector, embedder: this.#deps.embedder }, connector, {
+        const outcome = await upsertMediaFromSource({ vector: this.#deps.vector, embedder: this.#embedderFor(job) }, connector, {
             kind: media.kind,
             mime: media.mime,
             uri,
@@ -1024,12 +1036,14 @@ export class DeltaSyncRunner {
             return outcome.reason ? { ...skipped, mediaSkip: outcome.reason } : skipped;
         return {
             action: existing ? 'updated' : 'added',
-            chunksEmbedded: Math.max(0, outcome.pagesEmbedded - 1),
+            chunksEmbedded: outcome.chunksEmbedded,
             chunksSkipped: 0,
+            ...(outcome.embeddingDegraded ? { embeddingDegraded: outcome.embeddingDegraded } : {}),
         };
     }
-    async #upsertFromSource(job, connector, entry, indexedAtUri, moves) {
+    async #upsertFromSource(job, connector, entry, indexedAtUri, moves, binarySniffChars) {
         const skipped = { action: 'skipped', chunksEmbedded: 0, chunksSkipped: 0 };
+        const binary = (by) => ({ ...skipped, binarySkip: by });
         const uri = entry.uri;
         const filename = extractFilename(uri);
         if (!filename)
@@ -1040,7 +1054,7 @@ export class DeltaSyncRunner {
         if (media)
             return this.#upsertMedia(job, connector, entry, indexedAtUri, filename, media);
         if (isBinaryFile(filename))
-            return skipped;
+            return binary('extension');
         // Skip files larger than the size cap (live): `brainSyncMaxFileMb` (5 MB),
         // or `brainSyncMaxAppendFileMb` (20 MB) for an append-shaped file, whose
         // append re-embeds only the tail — never above the connector read cap
@@ -1052,6 +1066,8 @@ export class DeltaSyncRunner {
             return skipped;
         const readResult = await connector.read(entry.uri);
         const content = String(readResult.text ?? '');
+        if (looksBinaryContent(content, binarySniffChars))
+            return binary('content');
         const { contentHash: content_hash, provenPrefixChars, stored } = hashSyncedFile(filename, content, indexedAtUri?.payload);
         const now = this.#deps.clock.nowIso();
         // A new uri whose content is exactly ONE point that left disk in this pass
@@ -1074,201 +1090,20 @@ export class DeltaSyncRunner {
         const written = (action, chunks) => (movedFrom
             ? { action: 'moved', movedFrom, chunksEmbedded: chunks.embedded, chunksSkipped: chunks.skipped }
             : { action, chunksEmbedded: chunks.embedded, chunksSkipped: chunks.skipped });
-        const sourceChangedAt = entry.modifiedAt || now;
-        const restoringDraft = !movedFrom && existing?.payload.recovery_only === true
-            && existing.payload.status === 'pending_review' && existing.payload.content_hash === content_hash;
-        // If the file currently has a pending draft, do NOT overwrite draft content or undo state.
-        // Only refresh the last known upstream/source version metadata behind the file.
-        if (!movedFrom && existing?.payload?.status === 'pending_review' && !restoringDraft) {
-            const payload = {
-                ...existing.payload,
-                version: buildVersionInfo({
-                    previous: existing.payload.version,
-                    previousContent: existing.payload.content,
-                    nextContent: existing.payload.content,
-                    changedAt: existing.payload.version?.changedAt ?? existing.payload.updatedAt,
-                    sourceChangedAt,
-                    sourceVersion: content_hash,
-                    indexedAt: now,
-                    sync: {
-                        ...(existing.payload.version?.sync ?? {}),
-                        sourceUpdatedAt: entry.modifiedAt,
-                        sourceSize: entry.size,
-                        syncedAt: now,
-                        upstreamContentHash: content_hash,
-                        lastSyncedAgentId: job.agentId,
-                        status: 'stale',
-                    },
-                }),
-                status: 'pending_review',
-                updatedAt: now,
-            };
-            if (existing.payload.recovery_only === true) {
-                await this.#deps.vector.retainPayload({ id, payload });
-            }
-            else {
-                await this.#deps.vector.upsert({ id, payload });
-            }
-            return written('updated', { embedded: 0, skipped: 0 });
-        }
-        const enrichment = computeContributionEnrichment(entry.uri, content);
-        // The first reuse gate — an unchanged committed file only refreshes its
-        // payload. A forced re-embed skips it, and keeps that payload instead
-        // (below): an unchanged file must not read as changed.
-        const sameContent = !movedFrom && existing?.payload?.content_hash === content_hash && existing.payload.status === 'committed' && !existing.payload.recovery_only;
-        if (!job.force && sameContent) {
-            // Payload-only refresh BEFORE the content-hash skip: unchanged files
-            // still pick up new category/frontmatter/manifest enrichment after a
-            // deploy (the stored vector is kept). One sync pass migrates a source.
-            // A point that lost its inline content gets it back here — it is in hand.
-            const contentless = typeof existing.payload.content !== 'string';
-            const derived = buildEnrichmentRefresh(existing.payload, entry.uri, content, enrichment);
-            const refreshed = contentless ? { ...(derived ?? existing.payload), content: stored } : derived;
-            // A hash-matched point that has NOT drifted still needs the stamp, or it
-            // stays a sweep candidate forever: the sweep skips it (it is in the
-            // changed set) and the next full pass no longer sees it as changed. One
-            // payload-only write here closes it in the same pass.
-            if (refreshed || !isEnrichmentCurrent(existing.payload)) {
-                await this.#deps.vector.upsert({ id, payload: refreshed ?? existing.payload });
-            }
-            return skipped;
-        }
-        // The parent vector is keyed on its embedding TEXT: when the stored one was
-        // built from the same text (an append past the text cap, a move that kept
-        // its filename), the parent write below is payload-only.
-        const embeddingText = syncEmbeddingText(filename, content);
-        const embedParent = () => this.#deps.embedder.embedText(embeddingText, job.signal ? { signal: job.signal } : undefined);
-        // The second reuse gate — the stored parent vector was built from this
-        // exact text. A forced re-embed skips it too.
-        const reuseVector = !job.force && existing && !existing.payload.recovery_only ? storedVectorMatchesText(existing.payload, embeddingText) : false;
-        let embedded = null;
-        try {
-            if (!reuseVector)
-                embedded = await embedParent();
-        }
-        catch (error) {
-            // An embed outage (exhausted credits, a provider 429) must not freeze the
-            // existing point's DERIVED metadata: the index keeps carrying the OLD
-            // content, so category / frontmatter / manifest flags are re-derived from
-            // THAT content, payload-only (vector kept) — exactly what the enrichment
-            // sweep would do, which skips this file only because it sits in the
-            // changed set. The file's own failure is still reported by the caller;
-            // nothing here claims the new content was indexed.
-            if (!movedFrom && existing?.payload && typeof existing.payload.content === 'string') {
-                const stale = existing.payload.content;
-                const refreshed = buildEnrichmentRefresh(existing.payload, entry.uri, stale, computeContributionEnrichment(entry.uri, stale));
-                if (refreshed) {
-                    try {
-                        await this.#deps.vector.upsert({ id, payload: refreshed });
-                    }
-                    catch {
-                        // The embed failure is the error the caller reports; a failed
-                        // metadata refresh changes nothing about that.
-                    }
-                }
-            }
-            throw error;
-        }
-        // A forced re-embed of UNCHANGED content writes the stored payload back as
-        // it is (`updatedAt`, `version`, `createdAt`, …) with the inline text it
-        // embeds; only the vector and the boundary's `embed_model` stamp move.
-        const payload = (sameContent || restoringDraft) && existing ? { ...existing.payload, content: stored } : {
-            id,
-            uri,
-            folder_ancestors: uriAncestors(uri),
-            filename,
-            os_uri: resolveOsUri(uri, connector),
-            userId: job.userId,
-            projectId: job.projectId,
-            agentId: job.agentId,
-            ...(existing?.payload.userName ? { userName: existing.payload.userName } : {}),
-            // A payload-only write (the reused vector) keeps the stored marker; a
-            // vector write is re-stamped by the write boundary.
-            ...(existing?.payload.embed_model ? { embed_model: existing.payload.embed_model } : {}),
-            pinned: existing?.payload.pinned === true,
-            status: 'committed',
-            category: resolveSyncCategory(existing?.payload.category, entry.uri, content),
-            tags: existing?.payload.tags ?? [],
-            metadata: {
-                ...(existing?.payload.metadata ?? {}),
-            },
-            // Source-package enrichment (A3) — file points only, never chunks (F8).
-            ...(enrichment.manifest_usable ? { manifest_usable: true } : {}),
-            ...(enrichment.agent_owner ? { agent_owner: enrichment.agent_owner } : {}),
-            ...(enrichment.frontmatter ? { frontmatter: enrichment.frontmatter } : {}),
-            content: stored, // the indexed prefix; `content_hash` stays the WHOLE file's
-            content_hash,
-            createdAt: existing?.payload.createdAt ?? now,
-            updatedAt: now,
-            version: buildVersionInfo({
-                previous: existing?.payload.version,
-                previousContent: typeof existing?.payload.content === 'string' ? syncStoredText(existing.payload.content) : undefined,
-                nextContent: stored,
-                previousChangedAt: existing?.payload.version?.changedAt ?? existing?.payload.updatedAt,
-                changedAt: now,
-                sourceChangedAt,
-                sourceVersion: content_hash,
-                indexedAt: now,
-                sync: {
-                    sourceUpdatedAt: entry.modifiedAt,
-                    sourceSize: entry.size,
-                    syncedAt: now,
-                    upstreamContentHash: content_hash,
-                    lastSyncedAgentId: job.agentId,
-                    status: 'synced',
-                },
-            }),
+        const result = await syncIndexedFile({ vector: this.#deps.vector, embedder: this.#embedderFor(job) }, {
+            job, connector, entry, existing, id, filename, content, contentHash: content_hash,
+            stored, provenPrefixChars, now, movedFrom, indexedAtUri: Boolean(indexedAtUri),
+        });
+        return result.action === 'skipped' ? skipped : {
+            ...written(result.action, { embedded: result.chunksEmbedded, skipped: result.chunksSkipped }),
+            ...(result.embeddingDegraded ? { embeddingDegraded: result.embeddingDegraded } : {}),
         };
-        // Chunk ingestion BEFORE the parent upsert: the parent's content_hash is
-        // the convergence marker — if an abort lands mid-chunking, the parent
-        // still carries the OLD hash, so the next run reprocesses the file and
-        // the deterministic chunk ids converge. (Parent-first left a window where
-        // a hash-matched skip would never backfill the missing chunks.)
-        let chunksEmbedded = 0;
-        let chunksSkipped = 0;
-        if (content.length > 0) {
-            // The chunks inherit the fields the parent point carries (a carried
-            // payload keeps its own writer's identity, so its chunks do too).
-            const parentFields = {
-                userId: payload.userId,
-                projectId: payload.projectId,
-                agentId: payload.agentId,
-                uri: payload.uri,
-                folder_ancestors: payload.folder_ancestors,
-                filename: payload.filename,
-                os_uri: payload.os_uri,
-                status: 'committed',
-            };
-            const chunkUsage = {};
-            const chunkResult = await upsertChunksForContent({ vector: this.#deps.vector, embedder: this.#deps.embedder }, id, parentFields, content, entry.uri, chunkUsage, {
-                ...(job.signal ? { signal: job.signal } : {}),
-                ...(provenPrefixChars !== undefined && !job.force ? { appendOf: { prefixChars: provenPrefixChars } } : {}),
-                ...(job.force ? { force: true } : {}),
-            });
-            chunksEmbedded = chunkResult.embedded;
-            chunksSkipped = chunkResult.skipped;
-            // A moved point's chunks kept their vectors (same ids, same text); only
-            // their denormalized location follows the parent, payload-only.
-            if (movedFrom)
-                await relocateChunksForParent({ vector: this.#deps.vector }, id, parentFields);
-        }
-        if (embedded) {
-            await this.#deps.vector.upsert({ id, vector: embedded.embedding, payload });
-        }
-        else {
-            try {
-                await this.#deps.vector.upsert({ id, payload });
-            }
-            catch (error) {
-                // The text matched but no vector is stored under the id: embed after all.
-                if (!isVectorMissingError(error))
-                    throw error;
-                await this.#deps.vector.upsert({ id, vector: (await embedParent()).embedding, payload });
-            }
-        }
-        return written(indexedAtUri ? 'updated' : 'added', { embedded: chunksEmbedded, skipped: chunksSkipped });
     }
 }
+/** Read once per job, never per file (`brainSyncBinarySniffChars`, 0 = off). */
+function binarySniffCharsSetting() {
+    return Math.max(0, Math.floor(getBrainConfigNumber('brainSyncBinarySniffChars', undefined, BINARY_SNIFF_CHARS)));
+}
 function deterministicExternalNodeId(job, sourceRef) {
     const hash = createHash('sha1')
         .update(`${job.userId}:${job.projectId}:${job.source}:${sourceRef}`)