@futdevpro/fdp-agent-memory 0.1.0 → 1.1.14

package/build/src/_modules/ingest/_services/fam-ingest.control-service.js

@@ -4,13 +4,24 @@ exports.FAM_Ingest_ControlService = void 0;
 const tslib_1 = require("tslib");
 const crypto = tslib_1.__importStar(require("crypto"));
 const fs = tslib_1.__importStar(require("fs"));
+const path = tslib_1.__importStar(require("path"));
 const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
+const fam_table_type_enum_1 = require("../../../_enums/fam-table.type-enum");
 const error_codes_const_1 = require("../../../_collections/error-codes.const");
 const errors_control_service_1 = require("../../../_routes/server/errors/errors.control-service");
 const config_control_service_1 = require("../../../_routes/server/config/config.control-service");
 const fam_entry_data_model_1 = require("../../../_models/data-models/fam-entry.data-model");
+const fam_reference_code_util_1 = require("../../../_collections/fam-reference-code.util");
 const embedding_1 = require("../../embedding");
 const scope_reference_1 = require("../../scope-reference");
+const fam_content_hash_util_1 = require("../_collections/fam-content-hash.util");
+const fam_git_repo_util_1 = require("../_collections/fam-git-repo.util");
+const fam_project_identity_util_1 = require("../_collections/fam-project-identity.util");
+const fam_scan_path_util_1 = require("../_collections/fam-scan-path.util");
+const fam_scan_progress_util_1 = require("../_collections/fam-scan-progress.util");
+const fam_scan_summary_util_1 = require("../_collections/fam-scan-summary.util");
+const scope_reference_2 = require("../../scope-reference");
+const fam_reference_data_model_1 = require("../../../_models/data-models/fam-reference.data-model");
 const fam_chunker_control_service_1 = require("./fam-chunker.control-service");
 const fam_delta_compare_util_1 = require("./fam-delta-compare.util");
 const fam_ingest_run_data_service_1 = require("./fam-ingest-run.data-service");
@@ -36,6 +47,15 @@ const fam_scan_control_service_1 = require("./fam-scan.control-service");
  */
 class FAM_Ingest_ControlService {
     static _instance;
+    /**
+     * Bináris-tartalom heurisztika (FAM-REV-064): egy NUL-karakter (U+0000) az első ~8KB-ban → bináris
+     * (a UTF-8 szöveg sosem tartalmaz NUL-t). A denylist-en átcsúszott, ismeretlen-kiterjesztésű bináris
+     * fájl szűrője — így nem kerül szemét-chunk a generic chunkerbe. Pure + statikus (tesztelhető).
+     */
+    static isLikelyBinary(content) {
+        const sample = content.length > 8192 ? content.slice(0, 8192) : content;
+        return sample.indexOf(String.fromCharCode(0)) !== -1;
+    }
     /** Default issuer az ingest-műveletekhez. */
     issuer = 'FAM_Ingest_ControlService';
     static getInstance() {
@@ -90,6 +110,8 @@ class FAM_Ingest_ControlService {
             for (const chunk of chunks) {
                 if (chunk._id) {
                     await dataService.deleteData(chunk._id);
+                    // Pool-coherence (FAM-REV-046): a vektort az in-memory LVS-pool-ból is kivesszük.
+                    embedding_1.FAM_VectorSearch_ControlService.getInstance().removeVector(registryEntry.table, chunk._id);
                     deleted++;
                 }
             }
@@ -106,7 +128,9 @@ class FAM_Ingest_ControlService {
      */
     async runScan(request) {
         const issuer = request.issuer ?? this.issuer;
-        const runId = crypto.randomUUID();
+        // A pre-open (scope-resolve / discovery) hiba-summary-khoz kell egy id MIELŐTT a run-rekord
+        // megnyílna; a sikeres `open` UTÁN ezt a Dynamo-generált `_id`-re cseréljük (lásd lentebb).
+        let runId = crypto.randomUUID();
         const startedAt = Date.now();
         // (1) Scope reference-resolve (MP-3 `resolveForWrite`) — KÖTELEZŐ. Hiba → run `failed`.
         let canonicalScope;
@@ -132,6 +156,7 @@ class FAM_Ingest_ControlService {
                 include: request.include,
                 exclude: request.exclude,
                 tableOverride: request.tableOverride,
+                weights: request.weights,
             });
         }
         catch (error) {
@@ -140,25 +165,34 @@ class FAM_Ingest_ControlService {
                 error: error, scopePath: canonicalScope,
             });
         }
-        // (3) A run megnyitása (a chunkok ezzel az `ingestRunId`-vel hivatkoznak rá).
+        // (3) A run megnyitása (a chunkok ezzel az `ingestRunId`-vel hivatkoznak rá). A `_id`-t a Dynamo
+        // generálja (Mongo ObjectId — a `FAM_Scope` mintája; a client-generált UUID nem cast-olható
+        // `findById`-dal). Innentől a Dynamo-`_id` a runId.
         const runDataService = new fam_ingest_run_data_service_1.FAM_IngestRun_DataService({ issuer: issuer });
-        await runDataService.open({
-            runId: runId,
+        const openedRun = await runDataService.open({
             trigger: request.operation,
             table: discovery.tables,
             scopePath: canonicalScope,
             rootPath: request.path,
         });
+        runId = openedRun._id ?? runId;
         // (4) Per-fájl feldolgozás (flow-limit kötegekben; a per-fájl hiba NEM buktatja a run-t).
         const verdicts = { new: 0, modified: 0, equal: 0, deleted: 0 };
         const errors = [];
         let chunkCount = 0;
         const flowLimit = await this.resolveFlowLimit(canonicalScope);
-        for (let i = 0; i < discovery.routedFiles.length; i += flowLimit) {
+        const isDryRun = request.dryRun ?? false;
+        const total = discovery.routedFiles.length;
+        // Futás-közbeni progress (descriptive user feedback): a NAGY (workspace-szintű) scan haladását
+        // throttle-olt `[famIngest]` sorokkal jelezzük a konzolon (a `dryRun` NEM embeddel → ott nincs progress).
+        const progressIntervalMs = await this.resolveScanNumber('scan.progressIntervalMs', canonicalScope, 2000);
+        let lastProgressAt = startedAt;
+        for (let i = 0; i < total; i += flowLimit) {
             const batch = discovery.routedFiles.slice(i, i + flowLimit);
             const results = await Promise.all(batch.map((file) => this.processFile({
                 file: file, runId: runId, scopePath: canonicalScope,
-                dryRun: request.dryRun ?? false, issuer: issuer, errors: errors,
+                resolvedRoot: discovery.resolvedRoot,
+                dryRun: isDryRun, issuer: issuer, errors: errors,
             })));
             for (const result of results) {
                 if (!result) {
@@ -167,6 +201,42 @@ class FAM_Ingest_ControlService {
                 chunkCount += result.chunkCount;
                 this.tallyVerdicts(verdicts, result.deltaItems);
             }
+            // Throttle-olt progress-jelzés (a min. időköz a config; a végső 100% MINDIG kibocsátódik). dryRun-on
+            // kihagyva (nincs valódi embed-munka, a becslést a summary.estimate adja).
+            const processed = Math.min(i + flowLimit, total);
+            const now = Date.now();
+            if (!isDryRun && (processed >= total || now - lastProgressAt >= progressIntervalMs)) {
+                lastProgressAt = now;
+                fsm_dynamo_1.DyFM_Log.info(fam_scan_progress_util_1.FAM_ScanProgress_Util.progressLine({
+                    processed: processed, total: total, chunkCount: chunkCount,
+                    verdicts: verdicts, elapsedSeconds: (now - startedAt) / 1000,
+                }));
+            }
+        }
+        // (4b) Törölt-fájl reconciliation (whole-file orphan) — CSAK folder/project scan (a `scan-file`
+        // egyetlen fájlt céloz, nem reconciliál). A lemezről eltűnt fájlok chunkjai soft-delete-elődnek; a
+        // számuk a `verdicts.deleted`-be (a per-fájl, fájlon-belüli `deleted`-ek mellé).
+        if (request.operation !== 'scan-file') {
+            verdicts.deleted += await this.reconcileDeletedFiles({
+                discovery: discovery, scopePath: canonicalScope, issuer: issuer, dryRun: request.dryRun ?? false,
+            });
+        }
+        // (4c) File-rendszer összefoglaló (knowledge) — generált „mi van a projektben" entry a mappastruktúrából
+        // (folder/project scan, non-dryRun, van route-olt fájl). Best-effort: a hiba NEM buktatja a scant.
+        if (!isDryRun && request.operation !== 'scan-file' && discovery.routedFiles.length) {
+            try {
+                await this.persistScanSummary({ discovery: discovery, scopePath: canonicalScope, runId: runId, issuer: issuer });
+            }
+            catch (error) {
+                fsm_dynamo_1.DyFM_Log.warn(`[famIngest] fs-summary generálás sikertelen (a scan kész): ${error?.message}`);
+            }
+            // (4d) Projekt-identitás (git + package.json) → kereshető knowledge entry + reference-alias. Best-effort.
+            try {
+                await this.persistProjectIdentity({ discovery: discovery, scopePath: canonicalScope, runId: runId, issuer: issuer });
+            }
+            catch (error) {
+                fsm_dynamo_1.DyFM_Log.warn(`[famIngest] project-identity generálás sikertelen (a scan kész): ${error?.message}`);
+            }
         }
         // (5) A run lezárása (atomikus `$set`) + MON-esemény.
         const status = errors.length ? 'partial-failed' : 'ok';
@@ -182,6 +252,15 @@ class FAM_Ingest_ControlService {
             filesProcessed: discovery.routedFiles.length, filesSkipped: discovery.skippedFiles.length,
             chunkCount: chunkCount, verdicts: verdicts, durationMs: durationMs, status: status, errors: errors,
         };
+        // Pre-scan idő-becslés CSAK dryRun-on (descriptive user feedback): mennyi lesz a TÉNYLEGES scan embed-ideje.
+        if (isDryRun) {
+            const msPerChunk = await this.resolveScanNumber('scan.estimateMsPerChunk', canonicalScope, 250);
+            summary.estimate = fam_scan_progress_util_1.FAM_ScanProgress_Util.estimate({
+                chunksToEmbed: verdicts.new + verdicts.modified, msPerChunk: msPerChunk,
+            });
+            fsm_dynamo_1.DyFM_Log.info(`[famIngest] dryRun kész: ${total} fájl, ${chunkCount} chunk, `
+                + `${verdicts.new + verdicts.modified} embeddelendő → becsült idő ${summary.estimate.note}`);
+        }
         this.emitMonEvent(summary);
         return summary;
     }
@@ -194,6 +273,11 @@ class FAM_Ingest_ControlService {
         const table = set.file.route;
         try {
             const content = fs.readFileSync(set.file.absolutePath, 'utf-8');
+            // FAM-REV-064 bináris-tartalom guard: a denylist-en átcsúszott, ismeretlen-kiterjesztésű bináris
+            // (null-byte az első ~8KB-ban) NE kerüljön a generic chunkerbe (szemét-chunk ellen). Skip → 0 chunk.
+            if (FAM_Ingest_ControlService.isLikelyBinary(content)) {
+                return null;
+            }
             const chunks = await fam_chunker_control_service_1.FAM_Chunker_ControlService.getInstance().chunkFile({
                 content: content, relativePath: set.file.relativePath, table: table, scopePath: set.scopePath,
             });
@@ -202,13 +286,14 @@ class FAM_Ingest_ControlService {
                 throw new Error(`No store-registry entry for table '${table}'.`);
             }
             const dataService = new embedding_1.FAM_Entry_DataService({ dataParams: registryEntry.dataParams, issuer: set.issuer });
-            // Existing-by-key betöltés (a fájl nem soft-deleted chunkjai, scope-szűkítve).
-            const existing = await this.loadExistingChunks(dataService, set.file.relativePath, set.scopePath);
+            // Existing-by-key betöltés a KANONIKUS abszolút útvonalon (FAM-REV-050) — a scan-root-független
+            // fájl-identitás, így a delta bármely scan-gyökérről ugyanazt a fájlt matcheli (nincs duplikáció).
+            const existing = await this.loadExistingChunks(dataService, set.file.absolutePath, set.scopePath);
             const deltaItems = fam_delta_compare_util_1.FAM_DeltaCompare_Util.compare(chunks, existing);
             if (!set.dryRun) {
                 await this.applyVerdicts({
                     deltaItems: deltaItems, file: set.file, table: table, runId: set.runId,
-                    scopePath: set.scopePath, dataService: dataService,
+                    scopePath: set.scopePath, root: set.resolvedRoot, dataService: dataService,
                 });
             }
             return {
@@ -247,11 +332,47 @@ class FAM_Ingest_ControlService {
     async applyVerdicts(set) {
         for (const item of set.deltaItems) {
             if (item.verdict === 'equal') {
+                // Metaadat-backfill (FAM-REV-048 + FEAT-003): az equal-content entry `source.root`-ja ÉS
+                // `referenceCodes`-a frissül, ha eltér — a re-scan így a TELJES korpuszt root-tudatossá teszi
+                // (reconciliation-lefedettség) ÉS a kód-index-frissítést (pl. Tailwind-denylist-bővítés) a régi
+                // chunkokra is alkalmazza, RE-EMBED NÉLKÜL. Mindkettő FELTÉTELES (no-op, ha már stimmel),
+                // best-effort: a hiba NEM buktatja a scan-t (a content már naprakész).
+                if (item.existingId) {
+                    try {
+                        if (set.root) {
+                            await set.dataService.backfillSourceRoot(item.existingId, set.root);
+                        }
+                        // A default display-relatív út a friss scan-gyökérhez igazodjon (FAM-REV-050) —
+                        // egy korábban más root-reprezentációval (pl. scan-file basename) ingestelt chunk konzisztenssé.
+                        await set.dataService.backfillSourceFilePath(item.existingId, set.file.relativePath);
+                        if (item.chunk) {
+                            await set.dataService.backfillReferenceCodes(item.existingId, fam_reference_code_util_1.FAM_ReferenceCode_Util.extract(item.chunk.content));
+                            // Pozíció-backfill (FAM-REV-061): a chunker pozíció-fix a régi equal-chunkokra is,
+                            // re-embed nélkül (a friss chunk charStart/charEnd/sor-pozíciója felülír, ha eltér).
+                            await set.dataService.backfillPosition(item.existingId, item.chunk.position);
+                        }
+                    }
+                    catch {
+                        // best-effort metaadat-frissítés — a content már naprakész (equal), nem eszkaláljuk.
+                    }
+                    // FAM-REV-066: ha a meglévő equal-chunk NEM completed (transient embed-`error`/`pending`,
+                    // pl. LM Studio HTTP 500 alatt), ÚJRA-embeddeljük (a content egyezik, csak a vektor hiányzik)
+                    // → a tranziens embed-hibák a következő re-scankor AUTO-GYÓGYULNAK (nincs beragadt error).
+                    if (item.chunk && item.existingEmbeddingStatus && item.existingEmbeddingStatus !== 'completed') {
+                        await this.persistAndEmbed({
+                            chunk: item.chunk, existingId: item.existingId, contentHash: item.contentHash,
+                            file: set.file, table: set.table, runId: set.runId, scopePath: set.scopePath,
+                            root: set.root, dataService: set.dataService,
+                        });
+                    }
+                }
                 continue;
             }
             if (item.verdict === 'deleted') {
                 if (item.existingId) {
                     await set.dataService.deleteData(item.existingId);
+                    // Pool-coherence (FAM-REV-046): a törölt chunk vektora is ki a pool-ból.
+                    embedding_1.FAM_VectorSearch_ControlService.getInstance().removeVector(set.table, item.existingId);
                 }
                 continue;
             }
@@ -260,7 +381,7 @@ class FAM_Ingest_ControlService {
                 await this.persistAndEmbed({
                     chunk: item.chunk, existingId: item.existingId, contentHash: item.contentHash,
                     file: set.file, table: set.table, runId: set.runId, scopePath: set.scopePath,
-                    dataService: set.dataService,
+                    root: set.root, dataService: set.dataService,
                 });
             }
         }
@@ -278,13 +399,19 @@ class FAM_Ingest_ControlService {
             contentHash: set.contentHash,
             embeddingStatus: 'pending',
             scopePath: set.scopePath,
+            // Súly-kiemelés (FAM-REV-055): a scan-`weights`-ből feloldott fájl-súly (undefined → a tár model-default).
+            weight: set.file.weight,
             sourceFilePath: set.file.relativePath,
             chunkIndex: set.chunk.chunkIndex,
             chunkTotal: set.chunk.chunkTotal,
             chunkType: set.chunk.chunkType,
             position: set.chunk.position,
             headingPath: set.chunk.headingPath,
-            source: { type: 'scan', path: set.file.relativePath },
+            source: {
+                type: 'scan', path: set.file.relativePath, root: set.root, absolutePath: set.file.absolutePath,
+                // Git-repo provenance (user-FR): repoUrl + repoRelativePath + repoBranch → kattintható citáció.
+                ...fam_git_repo_util_1.FAM_GitRepo_Util.sourceFieldsFor(set.file.absolutePath),
+            },
             addedBy: 'scan',
             ingestRunId: set.runId,
         });
@@ -296,22 +423,218 @@ class FAM_Ingest_ControlService {
             table: set.table, entry: saved, callType: 'embed-write',
         });
     }
+    /**
+     * A file-rendszer-összefoglaló (`knowledge`, kind `fs-summary`) generálása + idempotens upsert-je a scan után.
+     * A `FAM_ScanSummary_Util` a mappastruktúrából építi a kereshető „mi van a projektben" szöveget. **Identitás:**
+     * `source.type='scan-summary'` + `source.absolutePath=<scan-gyökér>` + a levél-scope → re-scankor a MEGLÉVŐ-t
+     * frissíti (nincs duplikáció). **Idempotencia:** ha a `contentHash` változatlan (ugyanaz a fájl-lista), NO-OP
+     * (nincs fölös re-embed). A `scan-summary` source-type-ot a reconciliation (ami `scan`-t szűr) sosem érinti.
+     */
+    async persistScanSummary(set) {
+        const registryEntry = embedding_1.FAM_StoreRegistry_Util.getEntry(fam_table_type_enum_1.FAM_Table.knowledge);
+        if (!registryEntry) {
+            return;
+        }
+        const dataService = new embedding_1.FAM_Entry_DataService({ dataParams: registryEntry.dataParams, issuer: set.issuer });
+        const scopeLabel = set.scopePath.map((ref) => ref.canonicalName).join(' > ') || '(globális)';
+        const content = fam_scan_summary_util_1.FAM_ScanSummary_Util.build({
+            scopeLabel: scopeLabel,
+            root: set.discovery.resolvedRoot,
+            relativePaths: set.discovery.routedFiles.map((file) => file.relativePath),
+            skippedCount: set.discovery.skippedFiles.length,
+            tables: set.discovery.tables.map((table) => String(table)),
+        });
+        const contentHash = fam_content_hash_util_1.FAM_ContentHash_Util.hash(content);
+        // Meglévő összefoglaló a (source-type + scan-gyökér + levél-scope) identitáson.
+        const leafScopeId = set.scopePath.length ? set.scopePath[set.scopePath.length - 1].scopeId : undefined;
+        const filter = {};
+        filter['source.type'] = fam_scan_summary_util_1.FAM_ScanSummary_Util.SOURCE_TYPE;
+        filter['source.absolutePath'] = set.discovery.resolvedRoot;
+        filter.kind = fam_scan_summary_util_1.FAM_ScanSummary_Util.KIND; // ne ütközzön a project-identity-vel (azonos source-type + root + scope)
+        if (leafScopeId) {
+            filter['scopePath.scopeId'] = leafScopeId;
+        }
+        const existing = await dataService.findHydratableList(filter);
+        const existingEntry = existing[0];
+        // Változatlan fájl-lista → NO-OP (nincs fölös re-embed).
+        if (existingEntry && existingEntry.contentHash === contentHash) {
+            return;
+        }
+        const entry = new fam_entry_data_model_1.FAM_Entry({
+            _id: existingEntry?._id,
+            table: fam_table_type_enum_1.FAM_Table.knowledge,
+            content: content,
+            contentHash: contentHash,
+            embeddingStatus: 'pending',
+            scopePath: set.scopePath,
+            kind: fam_scan_summary_util_1.FAM_ScanSummary_Util.KIND,
+            tags: ['fs-summary', 'filesystem', 'scan-summary'],
+            weight: 2,
+            source: {
+                type: 'scan-summary', root: set.discovery.resolvedRoot, absolutePath: set.discovery.resolvedRoot,
+            },
+            addedBy: 'scan',
+            ingestRunId: set.runId,
+        });
+        const saved = await dataService.saveData(entry);
+        await embedding_1.FAM_EmbeddingPipeline_ControlService.getInstance().embedAndPersist({
+            table: fam_table_type_enum_1.FAM_Table.knowledge, entry: saved, callType: 'embed-write',
+        });
+        fsm_dynamo_1.DyFM_Log.info(`[famIngest] fs-summary ${existingEntry ? 'frissítve' : 'létrehozva'} `
+            + `(${scopeLabel}): ${set.discovery.routedFiles.length} fájl, ${set.discovery.tables.length} tár`);
+    }
+    /**
+     * A scan-gyökér PROJEKT-IDENTITÁSA (git + `package.json`) → (a) kereshető `knowledge` „project-identity"
+     * entry (név/verzió/leírás/függőségek/git-repo), (b) reference-alias (a package-név + rövid alakok → a
+     * projekt scope-ja). Idempotens (a knowledge a contentHash-en; a reference a scope-on). Best-effort.
+     */
+    async persistProjectIdentity(set) {
+        const identity = fam_project_identity_util_1.FAM_ProjectIdentity_Util.detect(set.discovery.resolvedRoot);
+        if (!fam_project_identity_util_1.FAM_ProjectIdentity_Util.hasIdentity(identity)) {
+            return;
+        }
+        const scopeLabel = set.scopePath.map((ref) => ref.canonicalName).join(' > ') || '(globális)';
+        const leaf = set.scopePath.length ? set.scopePath[set.scopePath.length - 1] : undefined;
+        // (a) knowledge „project-identity" entry — idempotens (kind + source-type + root + scope; contentHash-skip).
+        const registryEntry = embedding_1.FAM_StoreRegistry_Util.getEntry(fam_table_type_enum_1.FAM_Table.knowledge);
+        if (registryEntry) {
+            const dataService = new embedding_1.FAM_Entry_DataService({ dataParams: registryEntry.dataParams, issuer: set.issuer });
+            const content = fam_project_identity_util_1.FAM_ProjectIdentity_Util.buildContent(identity, scopeLabel, set.discovery.resolvedRoot);
+            const contentHash = fam_content_hash_util_1.FAM_ContentHash_Util.hash(content);
+            const filter = {};
+            filter['source.type'] = fam_scan_summary_util_1.FAM_ScanSummary_Util.SOURCE_TYPE;
+            filter['source.absolutePath'] = set.discovery.resolvedRoot;
+            filter.kind = fam_project_identity_util_1.FAM_ProjectIdentity_Util.KIND;
+            if (leaf) {
+                filter['scopePath.scopeId'] = leaf.scopeId;
+            }
+            const existing = await dataService.findHydratableList(filter);
+            const existingEntry = existing[0];
+            if (!existingEntry || existingEntry.contentHash !== contentHash) {
+                const entry = new fam_entry_data_model_1.FAM_Entry({
+                    _id: existingEntry?._id,
+                    table: fam_table_type_enum_1.FAM_Table.knowledge,
+                    content: content,
+                    contentHash: contentHash,
+                    embeddingStatus: 'pending',
+                    scopePath: set.scopePath,
+                    kind: fam_project_identity_util_1.FAM_ProjectIdentity_Util.KIND,
+                    tags: ['project-identity', 'package', ...(identity.isGit ? ['git'] : [])],
+                    weight: 3,
+                    source: {
+                        type: 'scan-summary', root: set.discovery.resolvedRoot, absolutePath: set.discovery.resolvedRoot,
+                    },
+                    addedBy: 'scan',
+                    ingestRunId: set.runId,
+                });
+                const saved = await dataService.saveData(entry);
+                await embedding_1.FAM_EmbeddingPipeline_ControlService.getInstance().embedAndPersist({
+                    table: fam_table_type_enum_1.FAM_Table.knowledge, entry: saved, callType: 'embed-write',
+                });
+            }
+        }
+        // (b) reference-alias (a package-név + rövid alakok → a projekt scope-ja) — string/exact-match feloldás.
+        if (leaf) {
+            await this.persistProjectReference(identity, leaf, set.discovery.resolvedRoot, set.issuer);
+        }
+        fsm_dynamo_1.DyFM_Log.info(`[famIngest] project-identity (${scopeLabel}): `
+            + `${identity.packageName ?? '(nincs package.json)'}${identity.isGit ? ' [git]' : ''}`
+            + `${identity.gitRepo ? ` ${identity.gitRepo}` : ''}, ${identity.dependencies.length} függőség`);
+    }
+    /**
+     * A projekt reference-aliasának idempotens upsert-je (a package-név + rövid alakok + repo → a projekt
+     * scope-ja, `category:'project-identity'`). A meglévőt a `canonicalScopeRef.scopeId`-n találja (re-scankor
+     * frissít, nem duplikál). A package-nevek exact/string-match-en oldódnak (nincs szükség embed-re).
+     */
+    async persistProjectReference(identity, leaf, dirRoot, issuer) {
+        const referenceDataService = new scope_reference_2.FAM_Reference_DataService({ issuer: issuer });
+        const filter = { category: 'project-identity' };
+        filter['canonicalScopeRef.scopeId'] = leaf.scopeId;
+        const existing = await referenceDataService.findDataList(filter);
+        const model = new fam_reference_data_model_1.FAM_Reference_DataModel({
+            _id: existing[0]?._id,
+            canonicalTerm: identity.packageName ?? leaf.canonicalName,
+            aliasTerms: fam_project_identity_util_1.FAM_ProjectIdentity_Util.buildAliases(identity, path.basename(dirRoot), leaf.canonicalName),
+            description: identity.description
+                ?? `A(z) '${leaf.canonicalName}' projekt (${identity.packageName ?? 'nincs package.json'}).`,
+            category: 'project-identity',
+            scopeLayer: leaf.layer,
+            canonicalScopeRef: leaf,
+        });
+        await referenceDataService.saveData(model);
+    }
     // =========================================================================
     // helpers
     // =========================================================================
     /**
      * A fájl meglévő (nem soft-deleted) chunkjai a táron + scope-on belül (`Map`-elésre a delta-
      * kulcshoz, SP-4.3). A `findHydratableList` az aktívakat adja (a soft-delete-elteket kihagyja).
-     * A scope-szűkítés a LEVÉL scopeId-jára (a write-path canonical scopePath-ot ír az entry-re).
+     * A fájl-identitás a KANONIKUS `source.absolutePath` (FAM-REV-050) — scan-root-független, így a
+     * delta bármely scan-gyökérről ugyanazt a fájlt matcheli. A scope-szűkítés a LEVÉL scopeId-jára.
      */
-    async loadExistingChunks(dataService, sourceFilePath, scopePath) {
-        const filter = { sourceFilePath: sourceFilePath };
+    async loadExistingChunks(dataService, absolutePath, scopePath) {
+        const filter = {};
+        filter['source.absolutePath'] = absolutePath;
         if (scopePath.length) {
             const leaf = scopePath[scopePath.length - 1];
             filter['scopePath.scopeId'] = leaf.scopeId;
         }
         return dataService.findHydratableList(filter);
     }
+    /**
+     * Törölt-fájl reconciliation (whole-file orphan, dsgn-004 §4.3 kiterjesztés). A scan-folder/scan-project
+     * UTÁN a táron maradt, de a lemezről ELTŰNT fájlok chunkjait soft-delete-eli (+ pool-removeVector). A
+     * fájlon-BELÜLi chunk-törlést a `FAM_DeltaCompare_Util` `deleted` verdiktje kezeli; EZ a teljes fájl
+     * eltűnését (amit az újra-scan különben sosem látna, mert a bejárás csak a meglévő fájlokat találja).
+     *
+     * **Subtree-tudatos a KANONIKUS abszolút úton (FAM-REV-050):** a present-set a most bejárt fájlok
+     * `absolutePath`-jai; orphan = az a `scan`-forrású, aktív entry, aminek az `source.absolutePath`-ja a
+     * jelen scan-gyökér (`resolvedRoot`) ALATT van (`isInside`), DE NINCS a present-set-ben → a fájl eltűnt
+     * a gyökér alól. A subtree-szűkítés MOST az absolutePath-on (`isInside`), NEM a `source.root`-egyezésen —
+     * így egy al-mappa scan a tágabb gyökérről ingestelt, de a saját subtree-jébe eső fájlokat is helyesen
+     * reconciliálja (és kívülre sosem nyúl). Az `absolutePath` nélküli (régi, migrálatlan) entry-ket kihagyja
+     * (no-migration safe). `dryRun` → csak számol. Visszaadja a reconciliált (soft-deleted) chunkok számát.
+     */
+    async reconcileDeletedFiles(set) {
+        const presentPaths = new Set();
+        for (const routed of set.discovery.routedFiles) {
+            presentPaths.add(routed.absolutePath);
+        }
+        for (const skipped of set.discovery.skippedFiles) {
+            presentPaths.add(skipped.absolutePath);
+        }
+        const leafScopeId = set.scopePath.length ? set.scopePath[set.scopePath.length - 1].scopeId : undefined;
+        let reconciled = 0;
+        for (const table of set.discovery.tables) {
+            const registryEntry = embedding_1.FAM_StoreRegistry_Util.getEntry(table);
+            if (!registryEntry) {
+                continue;
+            }
+            const dataService = new embedding_1.FAM_Entry_DataService({ dataParams: registryEntry.dataParams, issuer: set.issuer });
+            const filter = {};
+            filter['source.type'] = 'scan';
+            if (leafScopeId) {
+                filter['scopePath.scopeId'] = leafScopeId;
+            }
+            const entries = await dataService.findHydratableList(filter);
+            for (const entry of entries) {
+                const absolutePath = entry.source?.absolutePath;
+                // Migrálatlan (absolutePath nélküli) entry → kihagyjuk; csak a jelen gyökér ALATTI fájlokat nézzük.
+                if (!entry._id || !absolutePath
+                    || !fam_scan_path_util_1.FAM_ScanPath_Util.isInside(set.discovery.resolvedRoot, absolutePath)
+                    || presentPaths.has(absolutePath)) {
+                    continue;
+                }
+                // Orphan: a forrásfájl eltűnt a gyökér alól → soft-delete + pool-kivétel (FAM-REV-046 coherence).
+                if (!set.dryRun) {
+                    await dataService.deleteData(entry._id);
+                    embedding_1.FAM_VectorSearch_ControlService.getInstance().removeVector(table, entry._id);
+                }
+                reconciled++;
+            }
+        }
+        return reconciled;
+    }
     /** A verdikt-számok aggregálása a `FAM_IngestRun.verdicts`-be. */
     tallyVerdicts(verdicts, items) {
         for (const item of items) {
@@ -328,6 +651,11 @@ class FAM_Ingest_ControlService {
         const value = typeof resolved.value === 'number' ? resolved.value : 20;
         return Math.min(Math.max(value, 1), 200);
     }
+    /** Egy scan-szám-config feloldása a scope-kontextussal + fallback-kel (a `scan.estimateMsPerChunk` / `scan.progressIntervalMs`-hez). */
+    async resolveScanNumber(key, scopePath, fallback) {
+        const resolved = await config_control_service_1.FAM_Config_ControlService.getInstance().resolve(key, { scopePath: scopePath });
+        return typeof resolved.value === 'number' ? resolved.value : fallback;
+    }
     /**
      * A `failed` run-summary (a scope-resolve / felderítés bukásakor). A run rekord-mentés best-effort
      * (a hiba MÁR persistált); a hívónak a `failed` summary jelzi, hogy a scan nem futott le.
@@ -343,12 +671,16 @@ class FAM_Ingest_ControlService {
         };
         try {
             const runDataService = new fam_ingest_run_data_service_1.FAM_IngestRun_DataService({ issuer: this.issuer });
-            await runDataService.open({
-                runId: set.runId, trigger: set.request.operation, table: [],
+            // A `_id`-t a Dynamo generálja (ObjectId); a persistált run-rekordra a Dynamo-`_id`-vel
+            // hivatkozunk a `close`-ban (a client-UUID `findById`-dal nem cast-olható).
+            const openedRun = await runDataService.open({
+                trigger: set.request.operation, table: [],
                 scopePath: set.scopePath, rootPath: set.request.path,
             });
+            const persistedRunId = openedRun._id ?? set.runId;
+            summary.ingestRunId = persistedRunId;
             await runDataService.close({
-                runId: set.runId, table: [], filesProcessed: 0, filesSkipped: 0, chunkCount: 0,
+                runId: persistedRunId, table: [], filesProcessed: 0, filesSkipped: 0, chunkCount: 0,
                 verdicts: summary.verdicts, durationMs: summary.durationMs, status: 'failed', errors: summary.errors,
             });
         }

package/build/src/_modules/ingest/_services/fam-scan.control-service.js

@@ -4,11 +4,14 @@ exports.FAM_Scan_ControlService = void 0;
 const tslib_1 = require("tslib");
 const fs = tslib_1.__importStar(require("fs"));
 const path = tslib_1.__importStar(require("path"));
+const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
 const config_catalog_const_1 = require("../../../_collections/config-catalog.const");
 const config_control_service_1 = require("../../../_routes/server/config/config.control-service");
+const fam_famignore_util_1 = require("../_collections/fam-famignore.util");
 const fam_file_routing_util_1 = require("../_collections/fam-file-routing.util");
 const fam_glob_match_util_1 = require("../_collections/fam-glob-match.util");
 const fam_scan_path_util_1 = require("../_collections/fam-scan-path.util");
+const fam_scan_weight_util_1 = require("../_collections/fam-scan-weight.util");
 const fam_secret_exclude_util_1 = require("../_collections/fam-secret-exclude.util");
 /**
  * `FAM_Scan_ControlService` (SP-4.1, dsgn-004 §1/§2/§6) — a `write(scan-*)` fájl-felderítő + routing
@@ -41,9 +44,17 @@ class FAM_Scan_ControlService {
      */
     async discover(set) {
         const resolvedRoot = fam_scan_path_util_1.FAM_ScanPath_Util.resolveRoot(set.path, this.issuer);
-        const ignorePatterns = await this.resolveIgnorePatterns(set.scopePath, set.exclude);
+        // Effektív ignore: config `scan.ignorePatterns` + input `exclude` + a scan-GYÖKÉR `.fdpfamignore`-ja
+        // (in-repo, gitignore-szerű; folder/project scan-nél — a scan-file egyetlen explicit fájlt céloz). HOZZÁADÓDIK.
+        const baseIgnore = await this.resolveIgnorePatterns(set.scopePath, set.exclude);
+        const famIgnore = set.operation === 'scan-file' ? [] : fam_famignore_util_1.FAM_FamIgnore_Util.loadFromDir(resolvedRoot);
+        if (famIgnore.length) {
+            fsm_dynamo_1.DyFM_Log.info(`[famIngest] .fdpfamignore betöltve (${resolvedRoot}): ${famIgnore.length / 2} minta-sor`);
+        }
+        const ignorePatterns = [...baseIgnore, ...famIgnore];
         const maxFileSizeBytes = await this.resolveMaxFileSize(set.scopePath);
         const followSymlinks = await this.resolveFollowSymlinks(set.scopePath);
+        const testFileWeight = await this.resolveTestFileWeight(set.scopePath);
         const routedFiles = [];
         const skippedFiles = [];
         // A felderített abszolút fájlok (file = 1 db; folder/project = rekurzív bejárás).
@@ -61,6 +72,8 @@ class FAM_Scan_ControlService {
                 includePatterns: set.include,
                 maxFileSizeBytes: maxFileSizeBytes,
                 tableOverride: set.tableOverride,
+                weights: set.weights,
+                testFileWeight: testFileWeight,
             });
             if (routed.route === 'skip') {
                 skippedFiles.push(routed);
@@ -108,7 +121,12 @@ class FAM_Scan_ControlService {
         if (route === 'skip') {
             return { ...base, route: 'skip', skipReason: 'unrouted file-type' };
         }
-        return { ...base, route: route };
+        // (6) Súly-kiemelés (FAM-REV-055/058): explicit scan-`weights` → az; különben teszt-/spec-fájl → a default
+        // `scan.testFileWeight` (a próza-nehéz specek ne rangsoroljanak az implementáció fölé); különben model-default.
+        return {
+            ...base, route: route,
+            weight: fam_scan_weight_util_1.FAM_ScanWeight_Util.resolveWithDefault(set.relativePath, set.weights, set.testFileWeight),
+        };
     }
     /**
      * Rekurzív könyvtár-bejárás (a `scan-folder`/`scan-project` felderítője). Az ignore-patterns a
@@ -207,5 +225,10 @@ class FAM_Scan_ControlService {
         const resolved = await config_control_service_1.FAM_Config_ControlService.getInstance().resolve('scan.followSymlinks', { scopePath: scopePath });
         return typeof resolved.value === 'boolean' ? resolved.value : false;
     }
+    /** A teszt-/spec-fájlok default súlya a config-ból (`scan.testFileWeight`, default 0.5 — FAM-REV-058). */
+    async resolveTestFileWeight(scopePath) {
+        const resolved = await config_control_service_1.FAM_Config_ControlService.getInstance().resolve('scan.testFileWeight', { scopePath: scopePath });
+        return typeof resolved.value === 'number' ? resolved.value : 0.5;
+    }
 }
 exports.FAM_Scan_ControlService = FAM_Scan_ControlService;

package/build/src/_modules/ingest/index.js

@@ -12,7 +12,7 @@
  * (MVP1 csak fájlnév-/útvonal-alapú secret-exclude).
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.FAM_IngestRun_DataService = exports.FAM_ContentHash_Util = exports.FAM_DeltaCompare_Util = exports.FAM_SlidingChunker_Util = exports.FAM_MdChunker_Util = exports.FAM_TsChunker_Util = exports.FAM_Chunker_ControlService = exports.FAM_GlobMatch_Util = exports.FAM_SecretExclude_Util = exports.FAM_ScanPath_Util = exports.FAM_FileRouting_Util = exports.FAM_Scan_ControlService = exports.FAM_Ingest_ControlService = void 0;
+exports.FAM_IngestRun_DataService = exports.FAM_ContentHash_Util = exports.FAM_DeltaCompare_Util = exports.FAM_SlidingChunker_Util = exports.FAM_MdChunker_Util = exports.FAM_TsChunker_Util = exports.FAM_Chunker_ControlService = exports.FAM_GlobMatch_Util = exports.FAM_SecretExclude_Util = exports.FAM_ScanPath_Util = exports.FAM_ScanProgress_Util = exports.FAM_FileRouting_Util = exports.FAM_Scan_ControlService = exports.FAM_Ingest_ControlService = void 0;
 // SP-4.4 — ingest-orchestrátor (a `write(scan-*)` belépési pontja + group-by-run/delete-run)
 var fam_ingest_control_service_1 = require("./_services/fam-ingest.control-service");
 Object.defineProperty(exports, "FAM_Ingest_ControlService", { enumerable: true, get: function () { return fam_ingest_control_service_1.FAM_Ingest_ControlService; } });
@@ -21,6 +21,8 @@ var fam_scan_control_service_1 = require("./_services/fam-scan.control-service")
 Object.defineProperty(exports, "FAM_Scan_ControlService", { enumerable: true, get: function () { return fam_scan_control_service_1.FAM_Scan_ControlService; } });
 var fam_file_routing_util_1 = require("./_collections/fam-file-routing.util");
 Object.defineProperty(exports, "FAM_FileRouting_Util", { enumerable: true, get: function () { return fam_file_routing_util_1.FAM_FileRouting_Util; } });
+var fam_scan_progress_util_1 = require("./_collections/fam-scan-progress.util");
+Object.defineProperty(exports, "FAM_ScanProgress_Util", { enumerable: true, get: function () { return fam_scan_progress_util_1.FAM_ScanProgress_Util; } });
 var fam_scan_path_util_1 = require("./_collections/fam-scan-path.util");
 Object.defineProperty(exports, "FAM_ScanPath_Util", { enumerable: true, get: function () { return fam_scan_path_util_1.FAM_ScanPath_Util; } });
 var fam_secret_exclude_util_1 = require("./_collections/fam-secret-exclude.util");