@veewo/gitnexus 1.3.10 → 1.3.11

package/dist/core/kuzu/kuzu-adapter.js

@@ -1,10 +1,9 @@
 import fs from 'fs/promises';
-import { createReadStream } from 'fs';
-import { createInterface } from 'readline';
 import path from 'path';
 import kuzu from 'kuzu';
 import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, } from './schema.js';
 import { streamAllCSVsToDisk } from './csv-generator.js';
+import { streamRelationshipPairBucketsFromCsv } from './relationship-pair-buckets.js';
 let db = null;
 let conn = null;
 let currentDbPath = null;
@@ -146,45 +145,18 @@ export const loadGraphToKuzu = async (graph, repoPath, storagePath, onProgress)
         }
     }
     // Bulk COPY relationships — split by FROM→TO label pair (KuzuDB requires it)
-    // Stream-read the relation CSV line by line to avoid exceeding V8 max string length
-    let relHeader = '';
-    const relsByPair = new Map();
-    let skippedRels = 0;
-    let totalValidRels = 0;
-    await new Promise((resolve, reject) => {
-        const rl = createInterface({ input: createReadStream(csvResult.relCsvPath, 'utf-8'), crlfDelay: Infinity });
-        let isFirst = true;
-        rl.on('line', (line) => {
-            if (isFirst) {
-                relHeader = line;
-                isFirst = false;
-                return;
-            }
-            if (!line.trim())
-                return;
-            const match = line.match(/"([^"]*)","([^"]*)"/);
-            if (!match) {
-                skippedRels++;
-                return;
-            }
-            const fromLabel = getNodeLabel(match[1]);
-            const toLabel = getNodeLabel(match[2]);
-            if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
-                skippedRels++;
-                return;
-            }
-            const pairKey = `${fromLabel}|${toLabel}`;
-            let list = relsByPair.get(pairKey);
-            if (!list) {
-                list = [];
-                relsByPair.set(pairKey, list);
-            }
-            list.push(line);
-            totalValidRels++;
-        });
-        rl.on('close', resolve);
-        rl.on('error', reject);
+    // Stream relation CSV into per-pair temporary CSV files to avoid retaining
+    // all relationship lines in memory at once.
+    const pairBucketResult = await streamRelationshipPairBucketsFromCsv({
+        relCsvPath: csvResult.relCsvPath,
+        csvDir,
+        validTables,
+        getNodeLabel,
     });
+    const relHeader = pairBucketResult.relHeader;
+    const relsByPair = pairBucketResult.buckets;
+    const skippedRels = pairBucketResult.skippedRels;
+    const totalValidRels = pairBucketResult.totalValidRels;
     const insertedRels = totalValidRels;
     const warnings = [];
     let fallbackStats = { attempted: 0, succeeded: 0, failed: 0 };
@@ -192,15 +164,13 @@ export const loadGraphToKuzu = async (graph, repoPath, storagePath, onProgress)
         log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPair.size} types`);
         let pairIdx = 0;
         let failedPairEdges = 0;
-        const failedPairLines = [];
-        for (const [pairKey, lines] of relsByPair) {
+        const failedPairCsvPaths = [];
+        for (const [pairKey, bucket] of relsByPair) {
             pairIdx++;
             const [fromLabel, toLabel] = pairKey.split('|');
-            const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
-            await fs.writeFile(pairCsvPath, relHeader + '\n' + lines.join('\n'), 'utf-8');
-            const normalizedPath = normalizeCopyPath(pairCsvPath);
+            const normalizedPath = normalizeCopyPath(bucket.csvPath);
             const copyQuery = `COPY ${REL_TABLE_NAME} FROM "${normalizedPath}" (from="${fromLabel}", to="${toLabel}", HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;
-            if (pairIdx % 5 === 0 || lines.length > 1000) {
+            if (pairIdx % 5 === 0 || bucket.rowCount > 1000) {
                 log(`Loading edges: ${pairIdx}/${relsByPair.size} types (${fromLabel} -> ${toLabel})`);
             }
             try {
@@ -213,20 +183,31 @@ export const loadGraphToKuzu = async (graph, repoPath, storagePath, onProgress)
                 }
                 catch (retryErr) {
                     const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
-                    warnings.push(`${fromLabel}->${toLabel} (${lines.length} edges): ${retryMsg.slice(0, 80)}`);
-                    failedPairEdges += lines.length;
-                    failedPairLines.push(...lines);
+                    warnings.push(`${fromLabel}->${toLabel} (${bucket.rowCount} edges): ${retryMsg.slice(0, 80)}`);
+                    failedPairEdges += bucket.rowCount;
+                    failedPairCsvPaths.push(bucket.csvPath);
                 }
             }
+        }
+        if (failedPairCsvPaths.length > 0) {
+            const failedPairLines = [relHeader];
+            for (const failedPairPath of failedPairCsvPaths) {
+                const raw = await fs.readFile(failedPairPath, 'utf-8');
+                const lines = raw
+                    .split('\n')
+                    .slice(1)
+                    .filter(line => line.trim().length > 0);
+                failedPairLines.push(...lines);
+            }
+            log(`Inserting ${failedPairEdges} edges individually (missing schema pairs)`);
+            fallbackStats = await fallbackRelationshipInserts(failedPairLines, validTables, getNodeLabel);
+        }
+        for (const [, bucket] of relsByPair) {
             try {
-                await fs.unlink(pairCsvPath);
+                await fs.unlink(bucket.csvPath);
             }
             catch { }
         }
-        if (failedPairLines.length > 0) {
-            log(`Inserting ${failedPairEdges} edges individually (missing schema pairs)`);
-            fallbackStats = await fallbackRelationshipInserts([relHeader, ...failedPairLines], validTables, getNodeLabel);
-        }
     }
     // Cleanup all CSVs
     try {

package/dist/core/kuzu/relationship-pair-buckets.d.ts

@@ -0,0 +1,17 @@
+export interface RelationshipPairBucket {
+    csvPath: string;
+    rowCount: number;
+}
+export interface RelationshipPairBucketResult {
+    relHeader: string;
+    buckets: Map<string, RelationshipPairBucket>;
+    skippedRels: number;
+    totalValidRels: number;
+}
+export declare function bucketRelationshipLines(lines: string[], getNodeLabel: (id: string) => string): Promise<Map<string, string[]>>;
+export declare function streamRelationshipPairBucketsFromCsv(params: {
+    relCsvPath: string;
+    csvDir: string;
+    validTables: Set<string>;
+    getNodeLabel: (nodeId: string) => string;
+}): Promise<RelationshipPairBucketResult>;

package/dist/core/kuzu/relationship-pair-buckets.js

@@ -0,0 +1,79 @@
+import fs from 'fs/promises';
+import { createReadStream } from 'fs';
+import { createInterface } from 'readline';
+import path from 'path';
+const REL_ENDPOINTS_PATTERN = /"([^"]*)","([^"]*)"/;
+const parseRelationshipEndpoints = (line) => {
+    const match = line.match(REL_ENDPOINTS_PATTERN);
+    if (!match)
+        return null;
+    return { fromId: match[1], toId: match[2] };
+};
+export async function bucketRelationshipLines(lines, getNodeLabel) {
+    const buckets = new Map();
+    for (const line of lines) {
+        const endpoints = parseRelationshipEndpoints(line);
+        if (!endpoints)
+            continue;
+        const key = `${getNodeLabel(endpoints.fromId)}|${getNodeLabel(endpoints.toId)}`;
+        const rows = buckets.get(key) || [];
+        rows.push(line);
+        buckets.set(key, rows);
+    }
+    return buckets;
+}
+export async function streamRelationshipPairBucketsFromCsv(params) {
+    const { relCsvPath, csvDir, validTables, getNodeLabel } = params;
+    let relHeader = '';
+    const buckets = new Map();
+    let skippedRels = 0;
+    let totalValidRels = 0;
+    let isFirst = true;
+    let queue = Promise.resolve();
+    await new Promise((resolve, reject) => {
+        const rl = createInterface({ input: createReadStream(relCsvPath, 'utf-8'), crlfDelay: Infinity });
+        rl.on('line', (line) => {
+            queue = queue.then(async () => {
+                if (isFirst) {
+                    relHeader = line;
+                    isFirst = false;
+                    return;
+                }
+                if (!line.trim())
+                    return;
+                const endpoints = parseRelationshipEndpoints(line);
+                if (!endpoints) {
+                    skippedRels++;
+                    return;
+                }
+                const fromLabel = getNodeLabel(endpoints.fromId);
+                const toLabel = getNodeLabel(endpoints.toId);
+                if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
+                    skippedRels++;
+                    return;
+                }
+                const pairKey = `${fromLabel}|${toLabel}`;
+                let bucket = buckets.get(pairKey);
+                if (!bucket) {
+                    bucket = {
+                        csvPath: path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`),
+                        rowCount: 0,
+                    };
+                    buckets.set(pairKey, bucket);
+                    await fs.writeFile(bucket.csvPath, `${relHeader}\n`, 'utf-8');
+                }
+                await fs.appendFile(bucket.csvPath, `${line}\n`, 'utf-8');
+                bucket.rowCount++;
+                totalValidRels++;
+            }).catch((error) => {
+                rl.close();
+                reject(error);
+            });
+        });
+        rl.on('close', () => {
+            queue.then(() => resolve()).catch(reject);
+        });
+        rl.on('error', reject);
+    });
+    return { relHeader, buckets, skippedRels, totalValidRels };
+}

package/dist/core/kuzu/relationship-pair-buckets.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/core/kuzu/relationship-pair-buckets.test.js

@@ -0,0 +1,10 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { bucketRelationshipLines } from './relationship-pair-buckets.js';
+test('bucketRelationshipLines groups CSV lines by from/to pair without retaining all lines in one array', async () => {
+    const out = await bucketRelationshipLines([
+        '"Class:a","File:x","UNITY_RESOURCE_SUMMARY",1,"",0',
+        '"Class:a","CodeElement:b","UNITY_COMPONENT_INSTANCE",1,"",0',
+    ], (nodeId) => nodeId.split(':')[0]);
+    assert.deepEqual([...out.keys()].sort(), ['Class|CodeElement', 'Class|File']);
+});

package/dist/core/kuzu/schema.d.ts

@@ -11,7 +11,7 @@
 export declare const NODE_TABLES: readonly ["File", "Folder", "Function", "Class", "Interface", "Method", "CodeElement", "Community", "Process", "Struct", "Enum", "Macro", "Typedef", "Union", "Namespace", "Trait", "Impl", "TypeAlias", "Const", "Static", "Property", "Record", "Delegate", "Annotation", "Constructor", "Template", "Module"];
 export type NodeTableName = typeof NODE_TABLES[number];
 export declare const REL_TABLE_NAME = "CodeRelation";
-export declare const REL_TYPES: readonly ["CONTAINS", "DEFINES", "IMPORTS", "CALLS", "EXTENDS", "IMPLEMENTS", "MEMBER_OF", "STEP_IN_PROCESS", "UNITY_COMPONENT_IN", "UNITY_COMPONENT_INSTANCE", "UNITY_SERIALIZED_TYPE_IN"];
+export declare const REL_TYPES: readonly ["CONTAINS", "DEFINES", "IMPORTS", "CALLS", "EXTENDS", "IMPLEMENTS", "MEMBER_OF", "STEP_IN_PROCESS", "UNITY_COMPONENT_IN", "UNITY_COMPONENT_INSTANCE", "UNITY_RESOURCE_SUMMARY", "UNITY_SERIALIZED_TYPE_IN"];
 export type RelType = typeof REL_TYPES[number];
 export declare const EMBEDDING_TABLE_NAME = "CodeEmbedding";
 export declare const FILE_SCHEMA = "\nCREATE NODE TABLE File (\n  id STRING,\n  name STRING,\n  filePath STRING,\n  content STRING,\n  PRIMARY KEY (id)\n)";

package/dist/core/kuzu/schema.js

@@ -33,6 +33,7 @@ export const REL_TYPES = [
     'STEP_IN_PROCESS',
     'UNITY_COMPONENT_IN',
     'UNITY_COMPONENT_INSTANCE',
+    'UNITY_RESOURCE_SUMMARY',
     'UNITY_SERIALIZED_TYPE_IN',
 ];
 // ============================================================================

package/dist/core/unity/options.d.ts

@@ -1,2 +1,4 @@
 export type UnityResourcesMode = 'off' | 'on' | 'auto';
+export type UnityHydrationMode = 'parity' | 'compact';
 export declare function parseUnityResourcesMode(raw?: string): UnityResourcesMode;
+export declare function parseUnityHydrationMode(raw?: string): UnityHydrationMode;

package/dist/core/unity/options.js

@@ -7,3 +7,12 @@ export function parseUnityResourcesMode(raw) {
     }
     throw new Error('Invalid unity resources mode. Use off|on|auto.');
 }
+export function parseUnityHydrationMode(raw) {
+    if (!raw)
+        return 'compact';
+    const normalized = raw.trim().toLowerCase();
+    if (normalized === 'parity' || normalized === 'compact') {
+        return normalized;
+    }
+    throw new Error('Invalid unity hydration mode. Use parity|compact.');
+}

package/dist/core/unity/options.test.js

@@ -1,6 +1,6 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
-import { parseUnityResourcesMode } from './options.js';
+import { parseUnityHydrationMode, parseUnityResourcesMode } from './options.js';
 test('parseUnityResourcesMode defaults to off', () => {
     assert.equal(parseUnityResourcesMode(undefined), 'off');
 });
@@ -8,3 +8,10 @@ test('parseUnityResourcesMode validates mode', () => {
     assert.equal(parseUnityResourcesMode('on'), 'on');
     assert.throws(() => parseUnityResourcesMode('bad'), /unity resources mode/i);
 });
+test('parseUnityHydrationMode defaults to compact', () => {
+    assert.equal(parseUnityHydrationMode(undefined), 'compact');
+});
+test('parseUnityHydrationMode validates mode', () => {
+    assert.equal(parseUnityHydrationMode('compact'), 'compact');
+    assert.throws(() => parseUnityHydrationMode('bad'), /unity hydration mode/i);
+});

package/dist/core/unity/resolver.d.ts

@@ -5,6 +5,8 @@ export interface ResolveInput {
     repoRoot: string;
     symbol: string;
     scanContext?: UnityScanContext;
+    resourcePathAllowlist?: string[];
+    deepParseLargeResources?: boolean;
 }
 export interface UnityScalarField {
     name: string;
@@ -58,6 +60,7 @@ export interface ResolvedUnityBinding {
     resourceType: 'prefab' | 'scene' | 'asset';
     bindingKind: UnityBindingKind;
     componentObjectId: string;
+    lightweight?: boolean;
     evidence: UnityBindingEvidence;
     serializedFields: UnitySerializedFields;
     resolvedReferences: UnityResolvedReference[];

package/dist/core/unity/resolver.js

@@ -5,15 +5,24 @@ import { buildMetaIndex } from './meta-index.js';
 import { mergeOverrideChain } from './override-merger.js';
 import { findGuidHits } from './resource-hit-scanner.js';
 import { parseUnityYamlObjects } from './yaml-object-graph.js';
+const MAX_CACHED_RESOURCE_BYTES = 512 * 1024;
 export async function resolveUnityBindings(input) {
     const scriptPath = await resolveSymbolScriptPath(input.repoRoot, input.symbol, input.scanContext);
     const scriptGuid = await resolveScriptGuid(input.repoRoot, scriptPath, input.scanContext);
-    const hits = input.scanContext
+    const rawHits = input.scanContext
         ? (input.scanContext.guidToResourceHits.get(scriptGuid) ?? [])
         : await findGuidHits(input.repoRoot, scriptGuid);
+    const hits = applyResourceAllowlist(rawHits, input.resourcePathAllowlist);
     const resourceBindings = [];
     const unityDiagnostics = [];
+    const resourceSizeCache = new Map();
     for (const hit of hits) {
+        const shouldUseLightweightBinding = !input.deepParseLargeResources
+            && await isLargeResourceForDeepParse(input.repoRoot, hit.resourcePath, resourceSizeCache);
+        if (shouldUseLightweightBinding) {
+            resourceBindings.push(createLightweightBinding(hit));
+            continue;
+        }
         const blocks = await getResourceBlocks(input.repoRoot, hit.resourcePath, input.scanContext);
         const matchedComponents = blocks.filter((block) => block.objectType === 'MonoBehaviour' && block.fields.m_Script?.includes(scriptGuid));
         if (matchedComponents.length === 0) {
@@ -46,6 +55,44 @@ export async function resolveUnityBindings(input) {
         unityDiagnostics,
     };
 }
+function applyResourceAllowlist(hits, allowlist) {
+    if (!allowlist || allowlist.length === 0) {
+        return hits;
+    }
+    const normalizedAllowlist = new Set(allowlist.map((value) => normalizePath(value)));
+    return hits.filter((hit) => normalizedAllowlist.has(normalizePath(hit.resourcePath)));
+}
+function createLightweightBinding(hit) {
+    return {
+        resourcePath: hit.resourcePath,
+        resourceType: hit.resourceType,
+        bindingKind: hit.resourceType === 'scene' ? 'scene-override' : 'direct',
+        componentObjectId: `line-${hit.line}`,
+        lightweight: true,
+        evidence: {
+            line: hit.line,
+            lineText: hit.lineText,
+        },
+        serializedFields: {
+            scalarFields: [],
+            referenceFields: [],
+        },
+        resolvedReferences: [],
+        assetRefPaths: [],
+    };
+}
+async function isLargeResourceForDeepParse(repoRoot, resourcePath, cache) {
+    const normalizedPath = normalizePath(resourcePath);
+    const cached = cache.get(normalizedPath);
+    if (cached !== undefined) {
+        return cached;
+    }
+    const absolutePath = path.join(repoRoot, normalizedPath);
+    const stat = await fs.stat(absolutePath);
+    const isLarge = stat.size > MAX_CACHED_RESOURCE_BYTES;
+    cache.set(normalizedPath, isLarge);
+    return isLarge;
+}
 export function hasCoverage(resultSet) {
     return {
         hasScalar: resultSet.some((result) => result.serializedFields.scalarFields.length > 0),
@@ -102,9 +149,16 @@ async function getResourceBlocks(repoRoot, resourcePath, scanContext) {
         return cached;
     }
     const absoluteResourcePath = path.join(repoRoot, normalizedResourcePath);
+    let allowCache = Boolean(scanContext);
+    if (allowCache) {
+        const stat = await fs.stat(absoluteResourcePath);
+        allowCache = stat.size <= MAX_CACHED_RESOURCE_BYTES;
+    }
     const raw = await fs.readFile(absoluteResourcePath, 'utf-8');
     const blocks = parseUnityYamlObjects(raw);
-    scanContext?.resourceDocCache.set(normalizedResourcePath, blocks);
+    if (allowCache) {
+        scanContext?.resourceDocCache.set(normalizedResourcePath, blocks);
+    }
     return blocks;
 }
 function resolveBindingForComponent(componentBlock, blocks, hit, scanContext) {

package/dist/core/unity/resolver.test.js

@@ -188,6 +188,52 @@ test('resolveUnityBindings keeps existing scene serializedFields stable when .as
     assert.equal(mainUIDocument?.sourceLayer, 'scene');
     assert.equal(mainUIDocument?.guid, '44444444444444444444444444444444');
 });
+test('resolveUnityBindings supports resourcePathAllowlist filtering', async () => {
+    const result = await resolveUnityBindings({
+        repoRoot: fixtureRoot,
+        symbol: 'MainUIManager',
+        resourcePathAllowlist: ['Assets/Scene/NonExisting.unity'],
+    });
+    assert.equal(result.resourceBindings.length, 0);
+});
+test('resolveUnityBindings deepParseLargeResources can override lightweight fallback', async () => {
+    const tempRoot = await fs.mkdtemp(path.join(path.dirname(fixtureRoot), 'tmp-large-unity-'));
+    const scriptsDir = path.join(tempRoot, 'Assets/Scripts');
+    const sceneDir = path.join(tempRoot, 'Assets/Scene');
+    await fs.mkdir(scriptsDir, { recursive: true });
+    await fs.mkdir(sceneDir, { recursive: true });
+    try {
+        const scriptPath = 'Assets/Scripts/LargeSymbol.cs';
+        const scenePath = 'Assets/Scene/LargeScene.unity';
+        const scriptGuid = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
+        const padding = '#'.repeat(600 * 1024);
+        await fs.writeFile(path.join(tempRoot, scriptPath), 'public class LargeSymbol {}', 'utf-8');
+        await fs.writeFile(path.join(tempRoot, `${scriptPath}.meta`), `guid: ${scriptGuid}\n`, 'utf-8');
+        await fs.writeFile(path.join(tempRoot, scenePath), `--- !u!114 &11400000\nMonoBehaviour:\n  m_Script: {fileID: 11500000, guid: ${scriptGuid}, type: 3}\n  needPause: 1\n${padding}\n`, 'utf-8');
+        const scanContext = await buildUnityScanContext({
+            repoRoot: tempRoot,
+            scopedPaths: [scriptPath, `${scriptPath}.meta`, scenePath],
+            symbolDeclarations: [{ symbol: 'LargeSymbol', scriptPath }],
+        });
+        const lightweight = await resolveUnityBindings({
+            repoRoot: tempRoot,
+            symbol: 'LargeSymbol',
+            scanContext,
+        });
+        assert.equal(lightweight.resourceBindings[0]?.lightweight, true);
+        const expanded = await resolveUnityBindings({
+            repoRoot: tempRoot,
+            symbol: 'LargeSymbol',
+            scanContext,
+            deepParseLargeResources: true,
+        });
+        assert.equal(expanded.resourceBindings[0]?.lightweight, undefined);
+        assert.equal(expanded.resourceBindings[0]?.componentObjectId, '11400000');
+    }
+    finally {
+        await fs.rm(tempRoot, { recursive: true, force: true });
+    }
+});
 test('extractAssetRefPathReferences parses nested _relativePath rows and marks sprite assets', () => {
     const refs = extractAssetRefPathReferences({
         scalarFields: [

package/dist/core/unity/scan-context.d.ts

@@ -1,5 +1,6 @@
 import type { UnityResourceGuidHit } from './resource-hit-scanner.js';
 import type { UnityObjectBlock } from './yaml-object-graph.js';
+import type { UnityParitySeed } from '../ingestion/unity-parity-seed.js';
 export interface BuildScanContextInput {
     repoRoot: string;
     scopedPaths?: string[];
@@ -21,3 +22,7 @@ export interface UnityScanContext {
     resourceDocCache: Map<string, UnityObjectBlock[]>;
 }
 export declare function buildUnityScanContext(input: BuildScanContextInput): Promise<UnityScanContext>;
+export declare function buildUnityScanContextFromSeed(input: {
+    seed: UnityParitySeed;
+    symbolDeclarations?: UnitySymbolDeclaration[];
+}): UnityScanContext;