@rbalchii/anchor-engine 4.7.0

package/engine/dist/services/ingest/ingest-atomic.js

@@ -0,0 +1,437 @@
+import { db } from '../../core/db.js';
+import { config } from '../../config/index.js';
+import { filterTags } from '../../utils/tag-filter.js';
+export class AtomicIngestService {
+    async ingestResult(compound, molecules, atoms, buckets = ['core']) {
+        const startTime = Date.now();
+        const filename = compound.path.split(/[/\\]/).pop() || compound.path;
+        // Validate content lengths to prevent oversized atoms/molecules
+        const MAX_CONTENT_LENGTH = 500 * 1024; // 500KB limit
+        for (const mol of molecules) {
+            if (mol.content.length > MAX_CONTENT_LENGTH) {
+                console.warn(`[AtomicIngest] Warning: Molecule content exceeds maximum length (${mol.content.length} chars), truncating...`);
+                mol.content = mol.content.substring(0, MAX_CONTENT_LENGTH) + '... [TRUNCATED]';
+            }
+        }
+        for (const atom of atoms) {
+            if (atom.label.length > MAX_CONTENT_LENGTH) {
+                console.warn(`[AtomicIngest] Warning: Atom content exceeds maximum length (${atom.label.length} chars), truncating...`);
+                atom.label = atom.label.substring(0, MAX_CONTENT_LENGTH) + '... [TRUNCATED]';
+            }
+        }
+        console.log(`[AtomicIngest] ⏱️ START Persisting: ${filename} (${molecules.length} molecules, ${atoms.length} atoms)`);
+        // BUGFIX 2026-03-03: For large ingests (>100K molecules), temporarily drop FTS index
+        // to prevent PGlite corruption from concurrent index updates
+        const isLargeIngestion = molecules.length > 100000;
+        if (isLargeIngestion) {
+            console.log(`[AtomicIngest] ⏱️ Large ingestion detected (${molecules.length} molecules), dropping FTS index...`);
+            try {
+                await db.run('DROP INDEX IF EXISTS idx_molecules_content_gin');
+                await db.run('DROP INDEX IF EXISTS idx_atoms_content_gin');
+            }
+            catch (e) {
+                console.warn(`[AtomicIngest] Could not drop FTS indexes: ${e.message}`);
+            }
+        }
+        // Wrap entire ingestion in a transaction for atomicity and performance
+        await db.transaction(async () => {
+            await this._ingestResultInTransaction(compound, molecules, atoms, buckets);
+        });
+        // Recreate FTS indexes after bulk insert
+        if (isLargeIngestion) {
+            console.log(`[AtomicIngest] ⏱️ Recreating FTS indexes...`);
+            try {
+                await db.run(`
+                    CREATE INDEX IF NOT EXISTS idx_molecules_content_gin
+                    ON molecules
+                    USING GIN(to_tsvector('simple', content));
+                `);
+                await db.run(`
+                    CREATE INDEX IF NOT EXISTS idx_atoms_content_gin
+                    ON atoms
+                    USING GIN(to_tsvector('simple', content));
+                `);
+                // BUGFIX 2026-03-03: VACUUM ANALYZE after index rebuild to clear PGlite's cache
+                // This prevents "nfig"]D errors from stale query plans
+                await db.run('VACUUM ANALYZE atoms');
+                await db.run('VACUUM ANALYZE molecules');
+                console.log(`[AtomicIngest] ✅ FTS indexes recreated and optimized`);
+            }
+            catch (e) {
+                console.warn(`[AtomicIngest] Could not recreate FTS indexes: ${e.message}`);
+            }
+        }
+        const totalTime = ((Date.now() - startTime) / 1000).toFixed(2);
+        console.log(`[AtomicIngest] ✅ COMPLETE: ${filename} in ${totalTime}s`);
+    }
+    async _ingestResultInTransaction(compound, molecules, atoms, buckets) {
+        // 1. Persist Atoms (Tags)
+        const atomsStart = Date.now();
+        const uniqueAtomsMap = new Map();
+        for (const a of atoms) {
+            if (!uniqueAtomsMap.has(a.id)) {
+                uniqueAtomsMap.set(a.id, a);
+            }
+        }
+        const uniqueAtoms = Array.from(uniqueAtomsMap.values());
+        if (uniqueAtoms.length > 0) {
+            await this.batchWriteAtoms(uniqueAtoms);
+        }
+        console.log(`[AtomicIngest] ⏱️ Atoms persisted: ${Date.now() - atomsStart}ms`);
+        // 1.5. Persist Tags Table (The Nervous System)
+        const tagsStart = Date.now();
+        if (uniqueAtoms.length > 0) {
+            await this.batchWriteTags(uniqueAtoms, buckets);
+        }
+        console.log(`[AtomicIngest] ⏱️ Tags persisted: ${Date.now() - tagsStart}ms`);
+        // 2. Persist Molecules
+        const moleculesStart = Date.now();
+        if (molecules.length > 0) {
+            await this.batchWriteMolecules(molecules);
+        }
+        console.log(`[AtomicIngest] ⏱️ Molecules persisted: ${((Date.now() - moleculesStart) / 1000).toFixed(2)}s`);
+        // 3. Persist Atom Edges (Graph)
+        const edgesStart = Date.now();
+        if (compound.atoms && compound.atoms.length > 0) {
+            await this.batchWriteEdges(compound.id, compound.atoms);
+        }
+        console.log(`[AtomicIngest] ⏱️ Edges persisted: ${Date.now() - edgesStart}ms`);
+        // 4. Persist Compound (Atomic V4)
+        const compoundStart = Date.now();
+        await this.batchWriteCompounds([compound]);
+        console.log(`[AtomicIngest] ⏱️ Compound persisted: ${((Date.now() - compoundStart) / 1000).toFixed(2)}s`);
+        // 5. LEGACY BRIDGE: Populate 'atoms' table (was 'memory')
+        const memoryStart = Date.now();
+        await this.batchWriteMemory(compound, molecules, atoms, buckets);
+        console.log(`[AtomicIngest] ⏱️ Memory/Atoms persisted: ${((Date.now() - memoryStart) / 1000).toFixed(2)}s`);
+        // 6. Persist Atom Positions (Lazy Molecule Inflation)
+        const positionsStart = Date.now();
+        const atomLabelMap = new Map();
+        atoms.forEach(a => atomLabelMap.set(a.id, a.label));
+        await this.batchWriteAtomPositions(molecules, atomLabelMap);
+        console.log(`[AtomicIngest] ⏱️ Atom positions persisted: ${Date.now() - positionsStart}ms`);
+    }
+    async batchWriteAtoms(atoms) {
+        const chunkSize = 50;
+        for (let i = 0; i < atoms.length; i += chunkSize) {
+            const chunk = atoms.slice(i, i + chunkSize);
+            // Yield to event loop
+            if (i % 500 === 0 && i > 0)
+                await new Promise(resolve => setImmediate(resolve));
+            for (const atom of chunk) {
+                await db.run(`INSERT INTO atoms (id, content, source_path, timestamp, simhash, embedding, provenance, buckets, tags)
+                     VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+                     ON CONFLICT (id) DO UPDATE SET
+                       content = EXCLUDED.content,
+                       source_path = EXCLUDED.source_path,
+                       timestamp = EXCLUDED.timestamp,
+                       simhash = EXCLUDED.simhash,
+                       embedding = EXCLUDED.embedding,
+                       provenance = EXCLUDED.provenance,
+                       buckets = EXCLUDED.buckets,
+                       tags = EXCLUDED.tags`, [
+                    atom.id,
+                    atom.label, // label becomes content
+                    'atom_source', // source_path
+                    Date.now(), // timestamp
+                    "0", // simhash
+                    JSON.stringify(this.zeroVector()), // embedding
+                    'internal', // provenance
+                    ['atoms'], // buckets
+                    [atom.label] // tags
+                ]);
+            }
+        }
+    }
+    async batchWriteTags(atoms, buckets) {
+        const batchSize = 1000;
+        let batchValues = [];
+        let placeHolders = [];
+        const flush = async () => {
+            if (batchValues.length === 0)
+                return;
+            const query = `
+                INSERT INTO tags (atom_id, tag, bucket) 
+                VALUES ${placeHolders.join(', ')} 
+                ON CONFLICT DO NOTHING
+            `;
+            await db.run(query, batchValues);
+            batchValues = [];
+            placeHolders = [];
+        };
+        for (const atom of atoms) {
+            if (!atom.label)
+                continue;
+            // Ensure buckets are included
+            const allBuckets = new Set([...buckets, ...atom.buckets || []]);
+            // Filter atom label through blacklist
+            const filteredTags = filterTags([atom.label]);
+            const tags = filteredTags.length > 0 ? filteredTags : [atom.label]; // Keep original if filter removes everything
+            for (const bucket of allBuckets) {
+                for (const tag of tags) {
+                    batchValues.push(atom.id, tag, bucket);
+                    const offset = batchValues.length - 3;
+                    placeHolders.push(`($${offset + 1}, $${offset + 2}, $${offset + 3})`);
+                    if (placeHolders.length >= batchSize)
+                        await flush();
+                }
+            }
+        }
+        await flush();
+    }
+    // O(n/b) bulk INSERT — byte-budget batching to avoid PGlite WASM wire buffer overflow.
+    // PGlite has an ~1MB wire message limit; large chat molecules can overflow with fixed row counts.
+    async batchWriteMolecules(molecules) {
+        const MAX_BATCH_ROWS = 50;
+        const MAX_BATCH_BYTES = 512 * 1024; // 512KB per INSERT message (safe margin under PGlite's ~1MB limit)
+        const total = molecules.length;
+        const logInterval = Math.max(1000, Math.floor(total / 10));
+        const molStart = Date.now();
+        let i = 0;
+        while (i < molecules.length) {
+            // Build batch: add rows until we hit row limit OR byte budget
+            const batch = [];
+            let batchBytes = 0;
+            while (i < molecules.length && batch.length < MAX_BATCH_ROWS) {
+                const m = molecules[i];
+                const rowBytes = (m.content?.length ?? 0) + (m.id?.length ?? 0) + 200; // 200 = overhead estimate
+                if (batch.length > 0 && batchBytes + rowBytes > MAX_BATCH_BYTES)
+                    break;
+                batch.push(m);
+                batchBytes += rowBytes;
+                i++;
+            }
+            // Progress logging for large batches
+            if (total > 1000 && i % logInterval === 0 && i > 0) {
+                const elapsed = ((Date.now() - molStart) / 1000).toFixed(1);
+                const rate = Math.round(i / ((Date.now() - molStart) / 1000));
+                console.log(`[AtomicIngest] ⏱️ Molecules: ${((i / total) * 100).toFixed(0)}% (${i}/${total}) - ${elapsed}s elapsed, ${rate} mol/s`);
+            }
+            // Yield to event loop every 500 rows
+            if (i % 500 === 0 && i > 0) {
+                await new Promise(resolve => setImmediate(resolve));
+            }
+            // Build bulk INSERT with multiple VALUES
+            const placeholders = [];
+            const values = [];
+            let paramIdx = 1;
+            for (const m of batch) {
+                placeholders.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
+                values.push(m.id, m.content, m.compoundId, m.sequence, m.start_byte || 0, m.end_byte || 0, m.type || 'prose', 
+                // numeric_value: PostgreSQL real has range ~1E±37, clamp out-of-range to null
+                (m.numeric_value !== undefined && m.numeric_value !== null && Math.abs(m.numeric_value) < 1e37) ? m.numeric_value : null, m.numeric_unit || null, m.molecular_signature || '0', JSON.stringify(this.zeroVector()), // embedding (we don't compute embeddings here anymore)
+                m.timestamp || Date.now(), JSON.stringify(m.tags || []), JSON.stringify(m.entities || {}));
+            }
+            await db.run(`INSERT INTO molecules (id, content, compound_id, sequence, start_byte, end_byte, type, numeric_value, numeric_unit, molecular_signature, embedding, timestamp, tags, entities)
+                 VALUES ${placeholders.join(', ')}
+                 ON CONFLICT (id) DO UPDATE SET
+                   content = EXCLUDED.content,
+                   compound_id = EXCLUDED.compound_id,
+                   sequence = EXCLUDED.sequence,
+                   start_byte = EXCLUDED.start_byte,
+                   end_byte = EXCLUDED.end_byte,
+                   type = EXCLUDED.type,
+                   numeric_value = EXCLUDED.numeric_value,
+                   numeric_unit = EXCLUDED.numeric_unit,
+                   molecular_signature = EXCLUDED.molecular_signature,
+                   embedding = EXCLUDED.embedding,
+                   timestamp = EXCLUDED.timestamp,
+                   tags = EXCLUDED.tags,
+                   entities = EXCLUDED.entities`, values);
+        }
+        const molElapsed = ((Date.now() - molStart) / 1000).toFixed(1);
+        console.log(`[AtomicIngest] ⏱️ Molecules batch complete: ${total} molecules, ${molElapsed}s`);
+    }
+    async batchWriteEdges(compoundId, atomIds) {
+        const chunkSize = 50;
+        for (let i = 0; i < atomIds.length; i += chunkSize) {
+            const chunk = atomIds.slice(i, i + chunkSize);
+            // Yield if processing many edges
+            if (i % 500 === 0 && i > 0)
+                await new Promise(resolve => setImmediate(resolve));
+            for (const atomId of chunk) {
+                await db.run(`INSERT INTO edges (source_id, target_id, weight, relation)
+                     VALUES ($1, $2, $3, $4)
+                     ON CONFLICT (source_id, target_id, relation) DO UPDATE SET
+                       weight = EXCLUDED.weight,
+                       relation = EXCLUDED.relation`, [compoundId, atomId, 1.0, 'has_tag']);
+            }
+        }
+    }
+    async batchWriteCompounds(compounds) {
+        const chunkSize = 50;
+        // Limits to prevent WASM memory OOB - molecules are linked via compound_id anyway
+        const MAX_BODY_SIZE = 500 * 1024; // 500KB max for compound body
+        const MAX_MOLECULE_IDS = 10000; // Don't store more than 10K IDs
+        const MAX_ATOM_IDS = 1000; // Don't store more than 1K atom IDs
+        for (let i = 0; i < compounds.length; i += chunkSize) {
+            const chunk = compounds.slice(i, i + chunkSize);
+            for (const compound of chunk) {
+                // Truncate compound body if too large (full content in molecules)
+                let compoundBody = compound.compound_body || '';
+                if (compoundBody.length > MAX_BODY_SIZE) {
+                    compoundBody = compoundBody.substring(0, MAX_BODY_SIZE) + '\n... [TRUNCATED - see molecules for full content]';
+                }
+                // Limit array sizes (molecules have compound_id for lookups)
+                let atoms = compound.atoms || [];
+                let molecules = compound.molecules || [];
+                if (atoms.length > MAX_ATOM_IDS) {
+                    atoms = atoms.slice(0, MAX_ATOM_IDS);
+                }
+                if (molecules.length > MAX_MOLECULE_IDS) {
+                    molecules = molecules.slice(0, MAX_MOLECULE_IDS);
+                }
+                await db.run(`INSERT INTO compounds (id, compound_body, path, timestamp, provenance, molecular_signature, atoms, molecules, embedding)
+                     VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+                     ON CONFLICT (id) DO UPDATE SET
+                       compound_body = EXCLUDED.compound_body,
+                       path = EXCLUDED.path,
+                       timestamp = EXCLUDED.timestamp,
+                       provenance = EXCLUDED.provenance,
+                       molecular_signature = EXCLUDED.molecular_signature,
+                       atoms = EXCLUDED.atoms,
+                       molecules = EXCLUDED.molecules,
+                       embedding = EXCLUDED.embedding`, [
+                    compound.id,
+                    compoundBody,
+                    compound.path,
+                    compound.timestamp,
+                    compound.provenance,
+                    compound.molecular_signature || '0',
+                    JSON.stringify(atoms),
+                    JSON.stringify(molecules),
+                    JSON.stringify(this.zeroVector())
+                ]);
+            }
+        }
+    }
+    // O(n/b) bulk INSERT for atoms
+    async batchWriteMemory(compound, molecules, atoms, buckets) {
+        // 1. Write Compound Row
+        const MAX_ATOM_CONTENT_SIZE = 500 * 1024;
+        let atomContent = compound.compound_body;
+        if (atomContent.length > MAX_ATOM_CONTENT_SIZE) {
+            atomContent = atomContent.substring(0, MAX_ATOM_CONTENT_SIZE) + '... [TRUNCATED]';
+        }
+        await db.run(`INSERT INTO atoms (id, content, source_path, timestamp, simhash, embedding, provenance, buckets, tags, compound_id, start_byte, end_byte)
+             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+             ON CONFLICT (id) DO UPDATE SET
+               content = EXCLUDED.content,
+               source_path = EXCLUDED.source_path,
+               timestamp = EXCLUDED.timestamp,
+               simhash = EXCLUDED.simhash,
+               embedding = EXCLUDED.embedding,
+               provenance = EXCLUDED.provenance,
+               buckets = EXCLUDED.buckets,
+               tags = EXCLUDED.tags,
+               compound_id = EXCLUDED.compound_id,
+               start_byte = EXCLUDED.start_byte,
+               end_byte = EXCLUDED.end_byte`, [
+            compound.id,
+            atomContent,
+            compound.path,
+            compound.timestamp,
+            compound.molecular_signature || "0",
+            JSON.stringify(this.zeroVector()),
+            compound.provenance,
+            buckets,
+            atoms.map(a => a.label),
+            compound.id,
+            0,
+            compound.compound_body.length
+        ]);
+        // 2. Write Molecule Rows in Batches
+        const atomLabelMap = new Map();
+        atoms.forEach(a => atomLabelMap.set(a.id, a.label));
+        const batchSize = 50;
+        const total = molecules.length;
+        const logInterval = Math.max(1000, Math.floor(total / 10));
+        for (let i = 0; i < molecules.length; i += batchSize) {
+            const batch = molecules.slice(i, Math.min(i + batchSize, molecules.length));
+            // Progress logging
+            if (total > 1000 && i % logInterval === 0 && i > 0) {
+                console.log(`[AtomicIngest] ⏱️ Memory/Atoms: ${((i / total) * 100).toFixed(0)}% (${i}/${total})`);
+            }
+            if (i % 500 === 0 && i > 0)
+                await new Promise(resolve => setImmediate(resolve));
+            const placeholders = [];
+            const values = [];
+            let paramIdx = 1;
+            for (const m of batch) {
+                // Extract tags from atoms and filter through blacklist
+                const rawTags = (m.atoms || []).map(id => atomLabelMap.get(id)).filter(l => l !== undefined);
+                const specificTags = filterTags(rawTags);
+                placeholders.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
+                values.push(m.id, m.content, compound.path, m.timestamp || compound.timestamp, m.molecular_signature || this.generateHash(m.content), JSON.stringify(this.zeroVector()), compound.provenance, buckets, specificTags, m.compoundId, m.start_byte || 0, m.end_byte || 0);
+            }
+            await db.run(`INSERT INTO atoms (id, content, source_path, timestamp, simhash, embedding, provenance, buckets, tags, compound_id, start_byte, end_byte)
+                 VALUES ${placeholders.join(', ')}
+                 ON CONFLICT (id) DO UPDATE SET
+                   content = EXCLUDED.content,
+                   source_path = EXCLUDED.source_path,
+                   timestamp = EXCLUDED.timestamp,
+                   simhash = EXCLUDED.simhash,
+                   embedding = EXCLUDED.embedding,
+                   provenance = EXCLUDED.provenance,
+                   buckets = EXCLUDED.buckets,
+                   tags = EXCLUDED.tags,
+                   compound_id = EXCLUDED.compound_id,
+                   start_byte = EXCLUDED.start_byte,
+                   end_byte = EXCLUDED.end_byte`, values);
+        }
+    }
+    // Bulk INSERT for atom positions (Lazy Molecule Inflation)
+    async batchWriteAtomPositions(molecules, atomLabelMap) {
+        const batchSize = 100;
+        let batch = [];
+        // Iterate molecules and generate position rows on the fly
+        // This avoids creating a massive array of all position rows in memory
+        for (const m of molecules) {
+            if (!m.atoms || m.atoms.length === 0)
+                continue;
+            const midByte = m.start_byte !== undefined ? Math.floor((m.start_byte + (m.end_byte || m.start_byte)) / 2) : 0;
+            for (const atomId of m.atoms) {
+                const label = atomLabelMap.get(atomId);
+                if (label) {
+                    batch.push(m.compoundId, label, midByte);
+                    // If batch is full, flush it
+                    if (batch.length >= batchSize * 3) { // 3 params per row * batchSize
+                        await this.flushAtomPositionsBatch(batch);
+                        batch = [];
+                    }
+                }
+            }
+        }
+        // Flush remaining
+        if (batch.length > 0) {
+            await this.flushAtomPositionsBatch(batch);
+        }
+    }
+    async flushAtomPositionsBatch(flatValues) {
+        const rowsCount = flatValues.length / 3;
+        const placeholders = [];
+        let paramIdx = 1;
+        for (let i = 0; i < rowsCount; i++) {
+            placeholders.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
+        }
+        await db.run(`INSERT INTO atom_positions (compound_id, atom_label, byte_offset)
+             VALUES ${placeholders.join(', ')}
+             ON CONFLICT (compound_id, atom_label, byte_offset) DO NOTHING`, flatValues);
+    }
+    zeroVector() {
+        return new Array(config.MODELS.EMBEDDING_DIM).fill(0.1);
+    }
+    generateHash(str) {
+        // Simple hash for legacy field
+        let hash = 0;
+        if (str.length === 0)
+            return "0";
+        for (let i = 0; i < str.length; i++) {
+            const char = str.charCodeAt(i);
+            hash = ((hash << 5) - hash) + char;
+            hash |= 0;
+        }
+        return hash.toString(16);
+    }
+}
+//# sourceMappingURL=ingest-atomic.js.map

package/engine/dist/services/ingest/ingest-atomic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest-atomic.js","sourceRoot":"","sources":["../../../src/services/ingest/ingest-atomic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAEvD,MAAM,OAAO,mBAAmB;IAE5B,KAAK,CAAC,YAAY,CACd,QAAkB,EAClB,SAAqB,EACrB,KAAa,EACb,UAAoB,CAAC,MAAM,CAAC;QAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC,IAAI,CAAC;QAErE,gEAAgE;QAChE,MAAM,kBAAkB,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,cAAc;QAErD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;gBAC1C,OAAO,CAAC,IAAI,CAAC,oEAAoE,GAAG,CAAC,OAAO,CAAC,MAAM,wBAAwB,CAAC,CAAC;gBAC7H,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,iBAAiB,CAAC;YACnF,CAAC;QACL,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,gEAAgE,IAAI,CAAC,KAAK,CAAC,MAAM,wBAAwB,CAAC,CAAC;gBACxH,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,iBAAiB,CAAC;YACjF,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,QAAQ,KAAK,SAAS,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;QAEtH,qFAAqF;QACrF,6DAA6D;QAC7D,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;QACnD,IAAI,gBAAgB,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,+CAA+C,SAAS,CAAC,MAAM,oCAAoC,CAAC,CAAC;YACjH,IAAI,CAAC;gBACD,MAAM,EAAE,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;gBAC/D,MAAM,EAAE,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC/D,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;QAED,uEAAuE;QACvE,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,CAAC,0BAA0B,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,yCAAyC;QACzC,IAAI,gBAAgB,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;YAC3D,IAAI,CAAC;gBACD,MAAM,EAAE,CAAC,GAAG,CAAC;;;;iBAIZ,CAAC,CAAC;gBACH,MAAM,EAAE,CAAC,GAAG,CAAC;;;;iBAIZ,CAAC,CAAC;gBACH,gFAAgF;gBAChF,uDAAuD;gBACvD,MAAM,EAAE,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;gBACrC,MAAM,EAAE,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;gBACzC,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;YACxE,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,8BAA8B,QAAQ,OAAO,SAAS,GAAG,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,0BAA0B,CACpC,QAAkB,EAClB,SAAqB,EACrB,KAAa,EACb,OAAiB;QAEjB,0BAA0B;QAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,cAAc,GAAG,IAAI,GAAG,EAAgB,CAAC;QAC/C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACpB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC5B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QACD,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC,CAAC;QAExD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,sCAAsC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,IAAI,CAAC,CAAC;QAE/E,+CAA+C;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,qCAAqC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,IAAI,CAAC,CAAC;QAE7E,uBAAuB;QACvB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAC9C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE5G,gCAAgC;QAChC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,sCAAsC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,IAAI,CAAC,CAAC;QAE/E,kCAAkC;QAClC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,MAAM,IAAI,CAAC,mBAAmB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE1G,0DAA0D;QAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACjE,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE5G,sDAAsD;QACtD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAEpD,MAAM,IAAI,CAAC,uBAAuB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,+CAA+C,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,IAAI,CAAC,CAAC;IAChG,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAa;QACvC,MAAM,SAAS,GAAG,EAAE,CAAC;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5C,sBAAsB;YACtB,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YAEhF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,MAAM,EAAE,CAAC,GAAG,CACR;;;;;;;;;;4CAUwB,EACxB;oBACI,IAAI,CAAC,EAAE;oBACP,IAAI,CAAC,KAAK,EAAE,wBAAwB;oBACpC,aAAa,EAAE,cAAc;oBAC7B,IAAI,CAAC,GAAG,EAAE,EAAE,YAAY;oBACxB,GAAG,EAAE,UAAU;oBACf,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,YAAY;oBAC/C,UAAU,EAAE,aAAa;oBACzB,CAAC,OAAO,CAAC,EAAE,UAAU;oBACrB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO;iBACvB,CACJ,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,OAAiB;QACzD,MAAM,SAAS,GAAG,IAAI,CAAC;QACvB,IAAI,WAAW,GAAU,EAAE,CAAC;QAC5B,IAAI,YAAY,GAAa,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACrB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACrC,MAAM,KAAK,GAAG;;yBAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;;aAEnC,CAAC;YACF,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YACjC,WAAW,GAAG,EAAE,CAAC;YACjB,YAAY,GAAG,EAAE,CAAC;QACtB,CAAC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,KAAK;gBAAE,SAAS;YAC1B,8BAA8B;YAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,GAAI,IAAY,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;YAEzE,sCAAsC;YACtC,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,6CAA6C;YAEjH,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACrB,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;oBACvC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;oBACtC,YAAY,CAAC,IAAI,CAAC,KAAK,MAAM,GAAG,CAAC,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;oBACtE,IAAI,YAAY,CAAC,MAAM,IAAI,SAAS;wBAAE,MAAM,KAAK,EAAE,CAAC;gBACxD,CAAC;YACL,CAAC;QACL,CAAC;QACD,MAAM,KAAK,EAAE,CAAC;IAClB,CAAC;IAED,uFAAuF;IACvF,kGAAkG;IAC1F,KAAK,CAAC,mBAAmB,CAAC,SAAqB;QACnD,MAAM,cAAc,GAAG,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,mEAAmE;QACvG,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YAC1B,8DAA8D;YAC9D,MAAM,KAAK,GAAe,EAAE,CAAC;YAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBAC3D,MAAM,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,0BAA0B;gBACjG,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,QAAQ,GAAG,eAAe;oBAAE,MAAM;gBACvE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACd,UAAU,IAAI,QAAQ,CAAC;gBACvB,CAAC,EAAE,CAAC;YACR,CAAC;YAED,qCAAqC;YACrC,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,WAAW,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC5D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;gBAC9D,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,OAAO,cAAc,IAAI,QAAQ,CAAC,CAAC;YACxI,CAAC;YAED,qCAAqC;YACrC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YACxD,CAAC;YAED,yCAAyC;YACzC,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,MAAM,MAAM,GAAU,EAAE,CAAC;YACzB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACpB,YAAY,CAAC,IAAI,CAAC,KAAK,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,GAAG,CAAC,CAAC;gBACtP,MAAM,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,EACJ,CAAC,CAAC,OAAO,EACT,CAAC,CAAC,UAAU,EACZ,CAAC,CAAC,QAAQ,EACV,CAAC,CAAC,UAAU,IAAI,CAAC,EACjB,CAAC,CAAC,QAAQ,IAAI,CAAC,EACf,CAAC,CAAC,IAAI,IAAI,OAAO;gBACjB,8EAA8E;gBAC9E,CAAC,CAAC,CAAC,aAAa,KAAK,SAAS,IAAI,CAAC,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,EACxH,CAAC,CAAC,YAAY,IAAI,IAAI,EACtB,CAAC,CAAC,mBAAmB,IAAI,GAAG,EAC5B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,uDAAuD;gBAC1F,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,EACzB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,EAC5B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CACnC,CAAC;YACN,CAAC;YAED,MAAM,EAAE,CAAC,GAAG,CACR;0BACU,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;;;;;;gDAcD,EAChC,MAAM,CACT,CAAC;QACN,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,+CAA+C,KAAK,eAAe,UAAU,GAAG,CAAC,CAAC;IAClG,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,UAAkB,EAAE,OAAiB;QAC/D,MAAM,SAAS,GAAG,EAAE,CAAC;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,iCAAiC;YACjC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YAEhF,KAAK,MAAM,MAAM,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,GAAG,CACR;;;;oDAIgC,EAChC,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,CAAC,CACvC,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,SAAqB;QACnD,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,kFAAkF;QAClF,MAAM,aAAa,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,8BAA8B;QAChE,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAG,gCAAgC;QAClE,MAAM,YAAY,GAAG,IAAI,CAAC,CAAQ,oCAAoC;QAEtE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAEhD,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC3B,kEAAkE;gBAClE,IAAI,YAAY,GAAG,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAC;gBAChD,IAAI,YAAY,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;oBACtC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,GAAG,oDAAoD,CAAC;gBACnH,CAAC;gBAED,6DAA6D;gBAC7D,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;gBACjC,IAAI,SAAS,GAAG,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC;gBAEzC,IAAI,KAAK,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;oBAC9B,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;gBACzC,CAAC;gBACD,IAAI,SAAS,CAAC,MAAM,GAAG,gBAAgB,EAAE,CAAC;oBACtC,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;gBACrD,CAAC;gBAED,MAAM,EAAE,CAAC,GAAG,CACR;;;;;;;;;;sDAUkC,EAClC;oBACI,QAAQ,CAAC,EAAE;oBACX,YAAY;oBACZ,QAAQ,CAAC,IAAI;oBACb,QAAQ,CAAC,SAAS;oBAClB,QAAQ,CAAC,UAAU;oBACnB,QAAQ,CAAC,mBAAmB,IAAI,GAAG;oBACnC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;oBACrB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;oBACzB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;iBACpC,CACJ,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;IAED,+BAA+B;IACvB,KAAK,CAAC,gBAAgB,CAC1B,QAAkB,EAClB,SAAqB,EACrB,KAAa,EACb,OAAiB;QAEjB,wBAAwB;QACxB,MAAM,qBAAqB,GAAG,GAAG,GAAG,IAAI,CAAC;QACzC,IAAI,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC;QACzC,IAAI,WAAW,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;YAC7C,WAAW,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,qBAAqB,CAAC,GAAG,iBAAiB,CAAC;QACtF,CAAC;QAED,MAAM,EAAE,CAAC,GAAG,CACR;;;;;;;;;;;;;4CAagC,EAChC;YACI,QAAQ,CAAC,EAAE;YACX,WAAW;YACX,QAAQ,CAAC,IAAI;YACb,QAAQ,CAAC,SAAS;YAClB,QAAQ,CAAC,mBAAmB,IAAI,GAAG;YACnC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACjC,QAAQ,CAAC,UAAU;YACnB,OAAO;YACP,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;YACvB,QAAQ,CAAC,EAAE;YACX,CAAC;YACD,QAAQ,CAAC,aAAa,CAAC,MAAM;SAChC,CACJ,CAAC;QAEF,oCAAoC;QACpC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAEpD,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;QAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;QAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YAE5E,mBAAmB;YACnB,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,WAAW,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjD,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;YACtG,CAAC;YACD,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YAEhF,MAAM,YAAY,GAAa,EAAE,CAAC;YAClC,MAAM,MAAM,GAAU,EAAE,CAAC;YACzB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACpB,uDAAuD;gBACvD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,SAAS,CAAa,CAAC;gBACzG,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;gBAEzC,YAAY,CAAC,IAAI,CAAC,KAAK,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAEtN,MAAM,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,EACJ,CAAC,CAAC,OAAO,EACT,QAAQ,CAAC,IAAI,EACb,CAAC,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,EACjC,CAAC,CAAC,mBAAmB,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,EACrD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,EACjC,QAAQ,CAAC,UAAU,EACnB,OAAO,EACP,YAAY,EACZ,CAAC,CAAC,UAAU,EACZ,CAAC,CAAC,UAAU,IAAI,CAAC,EACjB,CAAC,CAAC,QAAQ,IAAI,CAAC,CAClB,CAAC;YACN,CAAC;YAED,MAAM,EAAE,CAAC,GAAG,CACR;0BACU,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;;;;gDAYD,EAChC,MAAM,CACT,CAAC;QACN,CAAC;IACL,CAAC;IAED,2DAA2D;IACnD,KAAK,CAAC,uBAAuB,CAAC,SAAqB,EAAE,YAAiC;QAC1F,MAAM,SAAS,GAAG,GAAG,CAAC;QACtB,IAAI,KAAK,GAAU,EAAE,CAAC;QAEtB,0DAA0D;QAC1D,sEAAsE;QACtE,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YACxB,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC/C,MAAM,OAAO,GAAG,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAE/G,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACvC,IAAI,KAAK,EAAE,CAAC;oBACR,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;oBAEzC,6BAA6B;oBAC7B,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC,CAAC,+BAA+B;wBAChE,MAAM,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;wBAC1C,KAAK,GAAG,EAAE,CAAC;oBACf,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,kBAAkB;QAClB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;QAC9C,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,uBAAuB,CAAC,UAAiB;QACnD,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,KAAK,QAAQ,EAAE,MAAM,QAAQ,EAAE,MAAM,QAAQ,EAAE,GAAG,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,EAAE,CAAC,GAAG,CACR;sBACU,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;2EAC8B,EAC/D,UAAU,CACb,CAAC;IACN,CAAC;IAGO,UAAU;QACd,OAAO,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC;IAEO,YAAY,CAAC,GAAW;QAC5B,+BAA+B;QAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC/B,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;YACnC,IAAI,IAAI,CAAC,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC;CACJ"}

package/engine/dist/services/ingest/ingest.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Ingest Service - Memory Ingestion with Provenance Tracking
+ *
+ * Implements the Data Provenance feature by adding a 'provenance' column
+ * to distinguish between "Sovereign" (User-Created) and "Ancillary" (External) data.
+ */
+interface IngestOptions {
+    atomize?: boolean;
+    skipCleaning?: boolean;
+}
+/**
+ * Ingest content into the memory database with provenance tracking
+ */
+export declare function ingestContent(content: string, source: string, type?: string, buckets?: string[], tags?: string[], options?: IngestOptions): Promise<{
+    status: string;
+    id?: string;
+    message?: string;
+}>;
+export interface IngestAtom {
+    id: string;
+    content: string;
+    sourceId: string;
+    sourcePath: string;
+    sequence: number;
+    timestamp: number;
+    provenance: 'internal' | 'external' | 'quarantine';
+    embedding?: number[];
+    hash?: string;
+    simhash?: string;
+    tags?: string[];
+    payload?: any;
+}
+/**
+ * Ingest pre-processed atoms
+ */
+/**
+ * Ingest pre-processed atoms (Batched)
+ */
+export declare function ingestAtoms(atoms: IngestAtom[], source: string, buckets?: string[], tags?: string[], // Batch-level tags (e.g., "inbox")
+fileTimestamp?: number): Promise<number>;
+/**
+ * Bulk import YAML content with provenance tracking
+ */
+export declare function importYamlContent(yamlContent: any[]): Promise<{
+    imported: number;
+    skipped: number;
+    errors: number;
+}>;
+export {};
+//# sourceMappingURL=ingest.d.ts.map

package/engine/dist/services/ingest/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../../src/services/ingest/ingest.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,UAAU,aAAa;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AA+BD;;GAEG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,MAAe,EACrB,OAAO,GAAE,MAAM,EAAa,EAC5B,IAAI,GAAE,MAAM,EAAO,EACnB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CA4F5D;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,UAAU,GAAG,UAAU,GAAG,YAAY,CAAC;IACnD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,GAAG,CAAC;CACf;AAED;;GAEG;AACH;;GAEG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,UAAU,EAAE,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,MAAM,EAAa,EAC5B,IAAI,GAAE,MAAM,EAAO,EAAE,mCAAmC;AACxD,aAAa,CAAC,EAAE,MAAM,GACrB,OAAO,CAAC,MAAM,CAAC,CAyGjB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAgC1H"}