@nac3/forge-cli 1.0.44 → 1.0.47

package/dist/semantic_graph/data_model.js

@@ -0,0 +1,288 @@
+/**
+ * data_model.ts -- data-model extraction for the semantic graph
+ * (Pablo 2026-06-23).
+ *
+ * The deriver indexed code modules + libraries + tooling, but NOT the
+ * project's DATA MODEL -- the tables/entities the app is actually about.
+ * In a large RPA/CRM project the schema IS the backbone (e.g. OSDE: the
+ * LLM had hand-noted "BD del demo HITL (6 tablas)" because the graph could
+ * not see them). This module discovers entities + their relationships from
+ * the common ways teams declare a schema, GENERICALLY (ships to customers;
+ * nothing hardcoded), and emits them as `datamodel` nodes + `references`
+ * edges so the panel/mobile graph + the prompt show the real domain model.
+ *
+ * Supported declarations (regex extraction, no per-language parser):
+ *   - SQL            : CREATE TABLE [schema.]name (...), FOREIGN KEY/REFERENCES.
+ *   - SQLAlchemy     : class X(Base) + __tablename__ + Column(...) + ForeignKey.
+ *   - Django ORM     : class X(models.Model) + <f> = models.ForeignKey(...).
+ *   - Pydantic/dataclass: class X(BaseModel) / @dataclass -> entity + fields.
+ *   - Prisma         : model X { ... } + relation fields.
+ *   - TypeORM        : @Entity() class X + @Column / @ManyToOne(() => Y).
+ *
+ * Bounded: a file/byte/entity cap keeps a pathological repo cheap; when a
+ * cap trips it is reported in stats (no silent truncation).
+ *
+ * ASCII-only.
+ */
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+const EXCLUDED_DIRS = new Set([
+    'node_modules', 'dist', 'build', 'out', 'coverage',
+    '__pycache__', '.pytest_cache', '.mypy_cache', '.ruff_cache',
+    'venv', '.venv', 'env', '.tox', 'site-packages', 'Lib',
+    '.git', '.svn', '.hg', '.next', '.expo', '.expo-shared',
+    '.gradle', '.idea', '.vscode', 'target', 'bin', 'obj',
+    '__snapshots__', '.cache',
+]);
+const SCHEMA_FILE_RE = /\.(sql|py|prisma|ts|js|rb)$/i;
+const MAX_FILES = 1200;
+const MAX_BYTES_PER_FILE = 600_000;
+const MAX_ENTITIES = 300;
+const MAX_FIELDS_SHOWN = 24;
+function dmId(name) {
+    return 'datamodel.' + name.replace(/[^a-z0-9_]/gi, '_').toLowerCase();
+}
+async function walkSchemaFiles(root) {
+    const out = [];
+    const walk = async (dir) => {
+        if (out.length >= MAX_FILES)
+            return;
+        let entries;
+        try {
+            entries = await fs.readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (out.length >= MAX_FILES)
+                return;
+            if (e.name.startsWith('.') || EXCLUDED_DIRS.has(e.name))
+                continue;
+            const full = path.join(dir, e.name);
+            if (e.isDirectory())
+                await walk(full);
+            else if (e.isFile() && SCHEMA_FILE_RE.test(e.name))
+                out.push(full);
+        }
+    };
+    await walk(root);
+    return out;
+}
+/** Extract CREATE TABLE statements (name + columns + references). */
+export function extractSqlTables(src, rel) {
+    const out = [];
+    const re = /create\s+table\s+(?:if\s+not\s+exists\s+)?[`"\[]?(?:[\w]+\.)?([\w]+)[`"\]]?\s*\(([\s\S]*?)\)\s*;/gi;
+    let m;
+    while ((m = re.exec(src)) !== null) {
+        const name = m[1];
+        const body = m[2] || '';
+        const fields = [];
+        const refs = new Set();
+        for (const rawLine of body.split(',')) {
+            const line = rawLine.trim();
+            if (!line)
+                continue;
+            const colM = line.match(/^[`"\[]?([\w]+)[`"\]]?\s+/);
+            const upper = line.toUpperCase();
+            if (colM && colM[1] && !/^(PRIMARY|FOREIGN|UNIQUE|CONSTRAINT|CHECK|KEY|INDEX)\b/.test(upper)) {
+                fields.push(colM[1]);
+            }
+            const refM = line.match(/references\s+[`"\[]?(?:[\w]+\.)?([\w]+)/i);
+            if (refM && refM[1])
+                refs.add(refM[1]);
+        }
+        out.push({ name, rel, flavor: 'sql', fields, refs: [...refs] });
+    }
+    return out;
+}
+/** Python: SQLAlchemy / Django / Pydantic / dataclass entities. */
+export function extractPythonEntities(src, rel) {
+    const out = [];
+    /* class X(Base|models.Model|BaseModel|...) or @dataclass\nclass X: */
+    const classRe = /(^|\n)[ \t]*(@dataclass[\s\S]{0,40}?)?class\s+([A-Za-z_]\w*)\s*(?:\(([^)]*)\))?\s*:/g;
+    let m;
+    while ((m = classRe.exec(src)) !== null) {
+        const decorator = m[2] || '';
+        const name = m[3];
+        const bases = (m[4] || '').toLowerCase();
+        const isOrm = /\bbase\b|models\.model|declarative|sqlmodel/.test(bases);
+        const isSchema = /basemodel|pydantic|sqlmodel/.test(bases);
+        const isDataclass = /@dataclass/.test(decorator);
+        if (!isOrm && !isSchema && !isDataclass)
+            continue;
+        /* body = until the next top-level `class ` / EOF */
+        const start = classRe.lastIndex;
+        const restMatch = src.slice(start).match(/\n(?=class\s|\S)/);
+        const bodyEnd = restMatch && restMatch.index != null ? start + restMatch.index : Math.min(src.length, start + 4000);
+        const body = src.slice(start, bodyEnd);
+        const fields = [];
+        const refs = new Set();
+        /* SQLAlchemy column / Django field / annotated attr */
+        for (const fm of body.matchAll(/(^|\n)[ \t]+([a-z_]\w*)\s*[:=]/gi)) {
+            const f = fm[2];
+            if (f.startsWith('__') || ['class', 'def', 'return', 'self'].includes(f))
+                continue;
+            if (!fields.includes(f))
+                fields.push(f);
+        }
+        /* relationships: ForeignKey('other.id'), relationship('Other'),
+         * models.ForeignKey(Other), ForeignKey(Other, ...) */
+        for (const rm of body.matchAll(/(?:ForeignKey|OneToOneField|ManyToManyField|relationship)\s*\(\s*['"]?([A-Za-z_][\w.]*)/g)) {
+            const target = rm[1].split('.')[0]; /* 'other.id' -> 'other' ; 'Other' -> 'Other' */
+            if (target && target.toLowerCase() !== name.toLowerCase())
+                refs.add(target);
+        }
+        out.push({ name, rel, flavor: isOrm ? 'sqlalchemy/django' : (isDataclass ? 'dataclass' : 'pydantic'), fields, refs: [...refs] });
+    }
+    return out;
+}
+/** Prisma schema: model X { field Type ... }. */
+export function extractPrismaModels(src, rel) {
+    const out = [];
+    const re = /model\s+([A-Za-z_]\w*)\s*\{([\s\S]*?)\}/g;
+    let m;
+    while ((m = re.exec(src)) !== null) {
+        const name = m[1];
+        const body = m[2] || '';
+        const fields = [];
+        const refs = new Set();
+        for (const line of body.split('\n')) {
+            const fm = line.trim().match(/^([A-Za-z_]\w*)\s+([A-Za-z_]\w*)(\[\])?/);
+            if (!fm)
+                continue;
+            const field = fm[1];
+            const type = fm[2];
+            if (field.startsWith('@'))
+                continue;
+            fields.push(field);
+            /* a field whose type is another model name (capitalised, not a scalar) */
+            if (/^[A-Z]/.test(type) && !['String', 'Int', 'Boolean', 'DateTime', 'Float', 'Json', 'Bytes', 'Decimal', 'BigInt'].includes(type)) {
+                if (type.toLowerCase() !== name.toLowerCase())
+                    refs.add(type);
+            }
+        }
+        out.push({ name, rel, flavor: 'prisma', fields, refs: [...refs] });
+    }
+    return out;
+}
+/** TypeORM: @Entity() class X + @Column + @ManyToOne(() => Y). */
+export function extractTypeOrmEntities(src, rel) {
+    if (!/@Entity\s*\(/.test(src))
+        return [];
+    const out = [];
+    const re = /@Entity\s*\([^)]*\)\s*(?:export\s+)?class\s+([A-Za-z_]\w*)([\s\S]*?)(?=\n@Entity|\nexport\s+class|\nclass\s|$)/g;
+    let m;
+    while ((m = re.exec(src)) !== null) {
+        const name = m[1];
+        const body = m[2] || '';
+        const fields = [];
+        const refs = new Set();
+        for (const cm of body.matchAll(/@Column\s*\([^)]*\)\s*([A-Za-z_]\w*)/g))
+            fields.push(cm[1]);
+        for (const rm of body.matchAll(/@(?:ManyToOne|OneToMany|ManyToMany|OneToOne)\s*\(\s*\(\)\s*=>\s*([A-Za-z_]\w*)/g)) {
+            if (rm[1] && rm[1].toLowerCase() !== name.toLowerCase())
+                refs.add(rm[1]);
+        }
+        out.push({ name, rel, flavor: 'typeorm', fields, refs: [...refs] });
+    }
+    return out;
+}
+function entitiesForFile(src, rel) {
+    const ext = rel.slice(rel.lastIndexOf('.') + 1).toLowerCase();
+    if (ext === 'sql')
+        return extractSqlTables(src, rel);
+    if (ext === 'prisma')
+        return extractPrismaModels(src, rel);
+    /* Code files: the language-specific extractor PLUS embedded SQL --
+     * Python/JS frequently ship the schema as CREATE TABLE strings inside
+     * the code (e.g. sqlite3 cursor.execute), which the ORM extractor would
+     * miss. Both run; extractDataModel dedupes by entity name. */
+    const out = [];
+    if (ext === 'py')
+        out.push(...extractPythonEntities(src, rel));
+    if (ext === 'ts' || ext === 'js')
+        out.push(...extractTypeOrmEntities(src, rel));
+    if (ext === 'py' || ext === 'ts' || ext === 'js') {
+        if (/create\s+table/i.test(src))
+            out.push(...extractSqlTables(src, rel));
+    }
+    return out;
+}
+/** Discover the project's data model -> datamodel nodes + reference edges. */
+export async function extractDataModel(projectRoot) {
+    const now = new Date().toISOString();
+    const files = await walkSchemaFiles(projectRoot);
+    const capped = files.length >= MAX_FILES;
+    /* name (lowercased) -> entity id, so refs resolve even across flavors. */
+    const byName = new Map();
+    for (const abs of files) {
+        let src;
+        try {
+            const st = await fs.stat(abs);
+            if (st.size > MAX_BYTES_PER_FILE)
+                continue;
+            src = await fs.readFile(abs, 'utf-8');
+        }
+        catch {
+            continue;
+        }
+        if (byName.size >= MAX_ENTITIES)
+            break;
+        const rel = path.relative(projectRoot, abs).replace(/\\/g, '/');
+        for (const ent of entitiesForFile(src, rel)) {
+            if (byName.size >= MAX_ENTITIES)
+                break;
+            const key = ent.name.toLowerCase();
+            const prev = byName.get(key);
+            /* merge duplicate declarations (e.g. SQL + ORM of the same table) */
+            if (prev) {
+                for (const f of ent.fields)
+                    if (!prev.fields.includes(f))
+                        prev.fields.push(f);
+                for (const r of ent.refs)
+                    if (!prev.refs.includes(r))
+                        prev.refs.push(r);
+            }
+            else {
+                byName.set(key, { ...ent });
+            }
+        }
+    }
+    const nodes = [];
+    for (const ent of byName.values()) {
+        const shown = ent.fields.slice(0, MAX_FIELDS_SHOWN);
+        const more = ent.fields.length > shown.length ? ' +' + (ent.fields.length - shown.length) + ' mas' : '';
+        const purpose = 'Entidad/tabla "' + ent.name + '" (' + ent.flavor + ', '
+            + ent.fields.length + ' campos'
+            + (shown.length ? ': ' + shown.join(', ') + more : '') + ') [' + ent.rel + ']';
+        nodes.push({
+            id: dmId(ent.name), kind: 'datamodel', name: ent.name,
+            source: 'data-model',
+            purpose: purpose.length > 200 ? purpose.slice(0, 197) + '...' : purpose,
+            solves: [], do_not_invent: [], api_surface: shown,
+            confidence: 'auto', weight: 0.75, added_at: now, updated_at: now,
+        });
+    }
+    const ids = new Set(nodes.map((n) => n.id));
+    const edgeMap = new Map();
+    let relations = 0;
+    for (const ent of byName.values()) {
+        const from = dmId(ent.name);
+        for (const ref of ent.refs) {
+            const to = dmId(ref);
+            if (to === from || !ids.has(to))
+                continue; /* no dangling */
+            const id = 'references:' + from + '->' + to;
+            if (!edgeMap.has(id)) {
+                edgeMap.set(id, { id, kind: 'references', from_id: from, to_id: to, weight: 0.7, source: 'data-model', created_at: now });
+                relations++;
+            }
+        }
+    }
+    return {
+        nodes, edges: [...edgeMap.values()],
+        stats: { entity_nodes: nodes.length, relation_edges: relations, capped },
+    };
+}
+//# sourceMappingURL=data_model.js.map

package/dist/semantic_graph/data_model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data_model.js","sourceRoot":"","sources":["../../src/semantic_graph/data_model.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,aAAa,GAAG,IAAI,GAAG,CAAS;IACpC,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU;IAClD,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,aAAa;IAC5D,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK;IACtD,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc;IACvD,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK;IACrD,eAAe,EAAE,QAAQ;CAC1B,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,8BAA8B,CAAC;AACtD,MAAM,SAAS,GAAG,IAAI,CAAC;AACvB,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,YAAY,GAAG,GAAG,CAAC;AACzB,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAgB5B,SAAS,IAAI,CAAC,IAAY;IACxB,OAAO,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,IAAY;IACzC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,KAAK,EAAE,GAAW,EAAiB,EAAE;QAChD,IAAI,GAAG,CAAC,MAAM,IAAI,SAAS;YAAE,OAAO;QACpC,IAAI,OAAmC,CAAC;QACxC,IAAI,CAAC;YAAC,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO;QAAC,CAAC;QACnF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,MAAM,IAAI,SAAS;gBAAE,OAAO;YACpC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;gBAAE,SAAS;YAClE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,CAAC,CAAC,WAAW,EAAE;gBAAE,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;iBACjC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrE,CAAC;IACH,CAAC,CAAC;IACF,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACjB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,GAAW;IACvD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,EAAE,GAAG,oGAAoG,CAAC;IAChH,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,wDAAwD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7F,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACpE,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC;gBAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,qBAAqB,CAAC,GAAW,EAAE,GAAW;IAC5D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,sEAAsE;IACtE,MAAM,OAAO,GAAG,sFAAsF,CAAC;IACvG,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QACnB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,6CAA6C,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAG,6BAA6B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ,IAAI,CAAC,WAAW;YAAE,SAAS;QAClD,oDAAoD;QACpD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC;QAChC,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,SAAS,IAAI,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC;QACpH,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,uDAAuD;QACvD,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;YACjB,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,SAAS;YACnF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD;8DACsD;QACtD,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,0FAA0F,CAAC,EAAE,CAAC;YAC3H,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAU,gDAAgD;YAC/F,IAAI,MAAM,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE;gBAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9E,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACnI,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,mBAAmB,CAAC,GAAW,EAAE,GAAW;IAC1D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,EAAE,GAAG,0CAA0C,CAAC;IACtD,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACxE,IAAI,CAAC,EAAE;gBAAE,SAAS;YAClB,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;YAAC,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;YAC1C,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YACpC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,0EAA0E;YAC1E,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnI,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE;oBAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,sBAAsB,CAAC,GAAW,EAAE,GAAW;IAC7D,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACzC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,EAAE,GAAG,iHAAiH,CAAC;IAC7H,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;QAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,uCAAuC,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC;QAC7F,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,iFAAiF,CAAC,EAAE,CAAC;YAClH,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE;gBAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,eAAe,CAAC,GAAW,EAAE,GAAW;IAC/C,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9D,IAAI,GAAG,KAAK,KAAK;QAAE,OAAO,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACrD,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,mBAAmB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC3D;;;kEAG8D;IAC9D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,GAAG,KAAK,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,sBAAsB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAChF,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QACjD,IAAI,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,WAAmB;IACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,WAAW,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,SAAS,CAAC;IACzC,0EAA0E;IAC1E,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,EAAE,CAAC,IAAI,GAAG,kBAAkB;gBAAE,SAAS;YAC3C,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YAAC,SAAS;QAAC,CAAC;QACrB,IAAI,MAAM,CAAC,IAAI,IAAI,YAAY;YAAE,MAAM;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAChE,KAAK,MAAM,GAAG,IAAI,eAAe,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;YAC5C,IAAI,MAAM,CAAC,IAAI,IAAI,YAAY;gBAAE,MAAM;YACvC,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7B,qEAAqE;YACrE,IAAI,IAAI,EAAE,CAAC;gBACT,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM;oBAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC9E,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI;oBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;wBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1E,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACxG,MAAM,OAAO,GAAG,iBAAiB,GAAG,GAAG,CAAC,IAAI,GAAG,KAAK,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI;cACpE,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,SAAS;cAC7B,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC;QACjF,KAAK,CAAC,IAAI,CAAC;YACT,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI;YACrD,MAAM,EAAE,YAAY;YACpB,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO;YACvE,MAAM,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,WAAW,EAAE,KAAK;YACjD,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG;SACjE,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC7C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC5B,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS,CAAK,iBAAiB;YAChE,MAAM,EAAE,GAAG,aAAa,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;YAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC1H,SAAS,EAAE,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACnC,KAAK,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,MAAM,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,EAAE;KACzE,CAAC;AACJ,CAAC"}

package/dist/semantic_graph/derive.d.ts

@@ -10,9 +10,12 @@ export interface DerivedGraph {
         module_nodes: number;
         lib_nodes: number;
         tooling_nodes: number;
+        datamodel_nodes: number;
         module_edges: number;
         lib_edges: number;
+        relation_edges: number;
         skipped_dangling: number;
+        capped: boolean;
     };
 }
 /** Tooling we discover GENERICALLY across any repo (this ships to

package/dist/semantic_graph/derive.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"derive.d.ts","sourceRoot":"","sources":["../../src/semantic_graph/derive.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAoGvD;;wDAEwD;AACxD,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAchE;AA+CD,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,KAAK,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9I;AAuDD;;;;;;;;;sCASsC;AACtC,wBAAsB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CA2F/E;AAoBD;sDACsD;AACtD,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B,GAAG,OAAO,CAAC,YAAY,CAAC,CAqIxB;AAED;;qDAEqD;AACrD,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAYtD"}
+{"version":3,"file":"derive.d.ts","sourceRoot":"","sources":["../../src/semantic_graph/derive.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAsJvD;;wDAEwD;AACxD,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAchE;AA+CD,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,KAAK,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;CAChN;AAuDD;;;;;;;;;sCASsC;AACtC,wBAAsB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CA2F/E;AAoBD;sDACsD;AACtD,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B,GAAG,OAAO,CAAC,YAAY,CAAC,CAiMxB;AAED;;qDAEqD;AACrD,wBAAsB,oBAAoB,CAAC,IAAI,EAAE;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAYtD"}

package/dist/semantic_graph/derive.js

@@ -21,6 +21,9 @@ import path from 'node:path';
 import { ingestFromPackageJson } from '../chat/semantic_graph_ingest.js';
 import { upsertNodes, upsertEdges, pruneDanglingEdges, pruneStaleDerivedNodes } from './store.js';
 import { extractImportSpecifiersAst } from './ast_extract.js';
+import { extractPythonImports } from './py_imports.js';
+import { extractLangImports } from './lang_imports.js';
+import { extractDataModel } from './data_model.js';
 /* PND-078 -- prefer the TS AST import walk (robust to comments/strings);
  * fall back to the regex for non-TS/JS files or on a parse failure. */
 const TS_JS_FILE = /\.(tsx?|jsx?|mjs|cjs)$/i;
@@ -120,6 +123,66 @@ async function collectLooseFiles(root, parent, add) {
             add(parent ? parent + '/' + e : e, e);
     }
 }
+/** Top-level dirs already handled by discoverModuleDirs/discoverLooseFiles
+ *  (their subtrees are scanned there). Everything ELSE in the tree is found
+ *  by discoverExtraSourceFiles so a project whose code lives outside these
+ *  (e.g. OSDE's demo_hitl/) is no longer invisible. */
+const FIXED_ROOTS = new Set(['src', 'packages', 'apps', 'app', 'components']);
+const MAX_EXTRA_FILE_NODES = 600;
+/** Recursively discover source FILES anywhere in the tree EXCEPT under the
+ *  fixed roots (handled elsewhere) and excluded dirs. One node per file so a
+ *  flat code dir (demo_hitl/ has 73 .py in a single folder) exposes its
+ *  files + their import edges instead of collapsing to one dot. Bounded by
+ *  MAX_EXTRA_FILE_NODES; `capped` tells the caller to surface the cut. */
+async function discoverExtraSourceFiles(projectRoot) {
+    const seen = new Set();
+    const files = [];
+    let capped = false;
+    const add = (rel, base) => {
+        if (files.length >= MAX_EXTRA_FILE_NODES) {
+            capped = true;
+            return;
+        }
+        const noExt = base.replace(/\.[^.]+$/, '');
+        const id = 'module.' + sanitiseId(noExt);
+        if (seen.has(id))
+            return;
+        seen.add(id);
+        files.push({ id, name: base, rel: rel.replace(/\\/g, '/') });
+    };
+    const walk = async (relDir) => {
+        if (files.length >= MAX_EXTRA_FILE_NODES) {
+            capped = true;
+            return;
+        }
+        const abs = relDir ? path.join(projectRoot, relDir) : projectRoot;
+        let entries;
+        try {
+            entries = await fs.readdir(abs, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (isExcludedDir(e.name))
+                continue;
+            const childRel = relDir ? relDir + '/' + e.name : e.name;
+            if (e.isDirectory()) {
+                /* skip the fixed roots at top level (handled elsewhere) */
+                if (!relDir && FIXED_ROOTS.has(e.name))
+                    continue;
+                await walk(childRel);
+            }
+            else if (e.isFile() && isSourceFile(e.name)) {
+                /* root-level loose files are already covered by discoverLooseFiles */
+                if (relDir)
+                    add(childRel, e.name);
+            }
+        }
+    };
+    await walk('');
+    return { files, capped };
+}
 /** npm/dir name -> id token (lowercase, alfanumerico + dashes). */
 function sanitiseId(name) {
     return name.replace(/^@/, '').replace(/[/]/g, '-').replace(/[^a-z0-9\-_]/gi, '-').toLowerCase();
@@ -175,7 +238,7 @@ async function walkSourceFiles(absDir) {
         }
         if (st.isDirectory())
             out.push(...await walkSourceFiles(full));
-        else if (/\.(ts|tsx|js|jsx|mjs)$/.test(e) && !e.endsWith('.d.ts'))
+        else if (isSourceFile(e))
             out.push(full);
     }
     return out;
@@ -462,11 +525,26 @@ export async function deriveProjectGraph(opts) {
     }
     /* Loose source files (incl. .py) as their own nodes -- the dir-based
      * scan above misses a project whose code is a flat file (e.g.
-     * src/robot.py). rel -> id so import edges can resolve to them. */
+     * src/robot.py). rel -> id so import edges can resolve to them.
+     * PLUS extra files anywhere outside the fixed roots (e.g. demo_hitl/),
+     * so a project whose code lives in a non-standard dir is no longer a
+     * handful of dots. */
     const looseFiles = await discoverLooseFiles(opts.project_root);
+    const extra = await discoverExtraSourceFiles(opts.project_root);
+    const looseSeen = new Set(looseFiles.map((f) => f.id));
+    for (const f of extra.files) {
+        if (!looseSeen.has(f.id)) {
+            looseSeen.add(f.id);
+            looseFiles.push(f);
+        }
+    }
     const looseByRel = new Map();
+    const byBasename = new Map(); /* file stem -> node id (Python resolution) */
     for (const lf of looseFiles) {
         looseByRel.set(lf.rel, lf.id);
+        const stem = (lf.name.replace(/\.[^.]+$/, '') || lf.name).toLowerCase();
+        if (!byBasename.has(stem))
+            byBasename.set(stem, lf.id);
         moduleNodes.push({
             id: lf.id, kind: 'module', name: lf.name,
             source: 'ast', purpose: 'Archivo ' + lf.name + ' (' + lf.rel + ').',
@@ -478,9 +556,14 @@ export async function deriveProjectGraph(opts) {
      * any repo so a customer's pipeline + helpers are DISCOVERED and land
      * in the prompt -- the model stops inventing build/deploy commands. */
     const toolingNodes = await discoverTooling(opts.project_root);
+    /* Data model (Pablo 2026-06-23): tables/entities + their relationships,
+     * extracted from SQL / ORMs / Prisma. The schema is the project's domain
+     * backbone; the graph was blind to it before. Self-contained nodes+edges
+     * (both ends are datamodel nodes). */
+    const dataModel = await extractDataModel(opts.project_root);
     const allNodeIds = new Set([
         ...mods.map((m) => m.id), ...libIds, ...looseFiles.map((f) => f.id),
-        ...toolingNodes.map((t) => t.id),
+        ...toolingNodes.map((t) => t.id), ...dataModel.nodes.map((n) => n.id),
     ]);
     function moduleForFile(absFile) {
         const rel = path.relative(opts.project_root, absFile).replace(/\\/g, '/');
@@ -498,6 +581,32 @@ export async function deriveProjectGraph(opts) {
         const pkg = spec.startsWith('@') ? spec.split('/').slice(0, 2).join('/') : (spec.split('/')[0] || spec);
         return 'lib.' + sanitiseId(pkg);
     }
+    /* Resolve a Python import to a project node id (or null). Handles
+     * relative (from .mod / from .. import x) by walking up from the file's
+     * dir, and absolute dotted (import a.b / from a.b import c) by trying the
+     * dotted path as a file/package, then falling back to a basename match.
+     * byBasename holds only real node ids, so a fallback can never dangle. */
+    function resolvePyImport(imp, fileAbs) {
+        const parts = imp.module.split('.').filter(Boolean);
+        if (parts.length === 0)
+            return null;
+        const tryPath = (baseAbs) => {
+            const asFile = moduleForFile(path.join(baseAbs, ...parts) + '.py');
+            if (asFile)
+                return asFile;
+            return moduleForFile(path.join(baseAbs, ...parts, '__init__.py'));
+        };
+        if (imp.dots > 0) {
+            let baseDir = path.dirname(fileAbs);
+            for (let i = 1; i < imp.dots; i++)
+                baseDir = path.dirname(baseDir);
+            return tryPath(baseDir) || byBasename.get(parts[parts.length - 1].toLowerCase()) || null;
+        }
+        return tryPath(opts.project_root)
+            || byBasename.get(parts[parts.length - 1].toLowerCase())
+            || byBasename.get(parts[0].toLowerCase())
+            || null;
+    }
     /* Aristas. */
     const edgeMap = new Map();
     let skipped = 0;
@@ -512,56 +621,68 @@ export async function deriveProjectGraph(opts) {
         if (!edgeMap.has(id))
             edgeMap.set(id, { id, kind, from_id: from, to_id: to, weight: 0.6, source: 'ast', created_at: now });
     }
-    for (const m of mods) {
-        const files = await walkSourceFiles(path.join(opts.project_root, m.dir));
-        for (const f of files) {
-            const owner = moduleForFile(f) || m.id;
-            let src;
-            try {
-                src = await fs.readFile(f, 'utf-8');
-            }
-            catch {
-                continue;
-            }
-            for (const spec of importSpecifiersFor(src, f)) {
-                if (spec.startsWith('node:') || spec.startsWith('bun:'))
-                    continue;
-                if (spec.startsWith('.') || spec.startsWith('/')) {
-                    const target = moduleForFile(path.resolve(path.dirname(f), spec));
-                    if (target)
-                        addEdge(owner, target, 'uses');
-                }
-                else {
-                    addEdge(owner, libIdFor(spec), 'depends_on');
-                }
-            }
+    /* Parse a file's imports (language-aware) and add resolved edges from
+     * ownerId. JS/TS uses the AST/regex specifier walk; .py uses the Python
+     * import grammar. Non-(JS|TS|PY) source files yield no specifiers. */
+    /* Resolve a relative import (Ruby require_relative / PHP require path) to a
+     * project node, trying the bare ref and the common code extensions. */
+    function resolveRelByExt(dirAbs, ref) {
+        for (const c of [ref, ref + '.rb', ref + '.php', ref + '.go', ref + '/index.php', ref + '/index.rb']) {
+            const t = moduleForFile(path.resolve(dirAbs, c));
+            if (t)
+                return t;
         }
+        return null;
     }
-    /* Loose-file imports (JS/TS specifiers only; .py yields none -- the
-     * Python-import grammar is not parsed here, so loose .py files appear
-     * as nodes without edges, which is correct for now). */
-    for (const lf of looseFiles) {
-        const abs = path.join(opts.project_root, lf.rel);
+    async function addEdgesForFile(absFile, ownerId) {
         let src;
         try {
-            src = await fs.readFile(abs, 'utf-8');
+            src = await fs.readFile(absFile, 'utf-8');
         }
         catch {
-            continue;
+            return;
+        }
+        if (/\.py$/i.test(absFile)) {
+            for (const imp of extractPythonImports(src)) {
+                const target = resolvePyImport(imp, absFile);
+                if (target)
+                    addEdge(ownerId, target, 'uses');
+            }
+            return;
         }
-        for (const spec of importSpecifiersFor(src, abs)) {
+        const ext = absFile.slice(absFile.lastIndexOf('.') + 1).toLowerCase();
+        if (ext === 'go' || ext === 'rb' || ext === 'php') {
+            for (const r of extractLangImports(src, ext)) {
+                const target = r.relative
+                    ? resolveRelByExt(path.dirname(absFile), r.ref)
+                    : (byBasename.get(r.ref.toLowerCase()) ?? null);
+                if (target)
+                    addEdge(ownerId, target, 'uses');
+            }
+            return;
+        }
+        for (const spec of importSpecifiersFor(src, absFile)) {
             if (spec.startsWith('node:') || spec.startsWith('bun:'))
                 continue;
             if (spec.startsWith('.') || spec.startsWith('/')) {
-                const target = moduleForFile(path.resolve(path.dirname(abs), spec));
+                const target = moduleForFile(path.resolve(path.dirname(absFile), spec));
                 if (target)
-                    addEdge(lf.id, target, 'uses');
+                    addEdge(ownerId, target, 'uses');
             }
             else {
-                addEdge(lf.id, libIdFor(spec), 'depends_on');
+                addEdge(ownerId, libIdFor(spec), 'depends_on');
             }
         }
     }
+    for (const m of mods) {
+        const files = await walkSourceFiles(path.join(opts.project_root, m.dir));
+        for (const f of files)
+            await addEdgesForFile(f, moduleForFile(f) || m.id);
+    }
+    /* Loose + extra files (incl. .py with real Python-import edges now). */
+    for (const lf of looseFiles) {
+        await addEdgesForFile(path.join(opts.project_root, lf.rel), lf.id);
+    }
     /* Los nodos de libreria del ingest no traen added_at/updated_at
      * (su shape es IngestedNode); normalizarlos para el store. */
     const libNodes = libIngest.map((l) => ({
@@ -569,17 +690,20 @@ export async function deriveProjectGraph(opts) {
         added_at: now,
         updated_at: now,
     }));
-    const edges = [...edgeMap.values()];
+    const edges = [...edgeMap.values(), ...dataModel.edges];
     return {
-        nodes: [...moduleNodes, ...libNodes, ...toolingNodes],
+        nodes: [...moduleNodes, ...libNodes, ...toolingNodes, ...dataModel.nodes],
         edges,
         stats: {
             module_nodes: moduleNodes.length,
             lib_nodes: libIngest.length,
             tooling_nodes: toolingNodes.length,
+            datamodel_nodes: dataModel.nodes.length,
             module_edges: edges.filter((e) => e.to_id.startsWith('module.')).length,
             lib_edges: edges.filter((e) => e.to_id.startsWith('lib.')).length,
+            relation_edges: dataModel.edges.length,
             skipped_dangling: skipped,
+            capped: extra.capped || dataModel.stats.capped,
         },
     };
 }