modscape 1.3.0 → 2.0.0

package/README.ja.md

@@ -270,7 +270,6 @@ modscape export ./models -o docs/ARCHITECTURE.md
 
 Modscape は以下の素晴らしいオープンソースプロジェクトによって支えられています：
 
-- [React Flow](https://reactflow.dev/) - インタラクティブなグラフ UI フレームワーク。
 - [CodeMirror 6](https://codemirror.net/) - 次世代のウェブベース・コードエディタ。
 - [Dagre](https://github.com/dagrejs/dagre) - 階層型グラフ・レイアウトエンジン。
 - [Lucide React](https://lucide.dev/) - シンプルで美しいアイコンセット。

package/README.md

@@ -261,7 +261,6 @@ modscape export ./models -o docs/ARCHITECTURE.md
 
 Modscape is made possible by these incredible open-source projects:
 
-- [React Flow](https://reactflow.dev/) - Interactive node-based UI framework.
 - [CodeMirror 6](https://codemirror.net/) - Next-generation code editor for the web.
 - [Dagre](https://github.com/dagrejs/dagre) - Directed graph layout engine.
 - [Lucide React](https://lucide.dev/) - Beautifully simple pixel-perfect icons.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "modscape",
-  "version": "1.3.0",
+  "version": "2.0.0",
   "description": "Modscape: A YAML-driven data modeling visualizer CLI",
   "repository": {
     "type": "git",

package/src/export.js

@@ -67,16 +67,13 @@ function generateMermaidLineage(schema) {
   let mermaid = 'graph TD\n';
   let hasLineage = false;
 
-  schema.tables.forEach(table => {
-    if (table.lineage?.upstream && table.lineage.upstream.length > 0) {
-      hasLineage = true;
-      const targetName = sanitize(table.name);
-      table.lineage.upstream.forEach(upId => {
-        const sourceTable = schema.tables.find(t => t.id === upId);
-        const sourceName = sanitize(sourceTable?.name || upId);
-        mermaid += `    ${sourceName} --> ${targetName}\n`;
-      });
-    }
+  (schema.lineage || []).forEach(edge => {
+    hasLineage = true;
+    const sourceTable = schema.tables.find(t => t.id === edge.from);
+    const targetTable = schema.tables.find(t => t.id === edge.to);
+    const sourceName = sanitize(sourceTable?.name || edge.from);
+    const targetName = sanitize(targetTable?.name || edge.to);
+    mermaid += `    ${sourceName} --> ${targetName}\n`;
   });
 
   return hasLineage ? mermaid : null;
@@ -197,10 +194,11 @@ export function generateMarkdown(schema, modelName) {
     }
 
     // Lineage
-    if (table.lineage?.upstream?.length > 0) {
-      const upstreamNames = table.lineage.upstream.map(upId => {
-        const t = schema.tables.find(t => t.id === upId);
-        return t ? t.name : upId;
+    const upstreamEdges = (schema.lineage || []).filter(e => e.to === table.id);
+    if (upstreamEdges.length > 0) {
+      const upstreamNames = upstreamEdges.map(e => {
+        const t = schema.tables.find(t => t.id === e.from);
+        return t ? t.name : e.from;
       });
       md += `**Upstream**: ${upstreamNames.map(n => `\`${n}\``).join(' → ')}\n\n`;
     }

package/src/import-dbt.js

@@ -34,6 +34,7 @@ export async function importDbt(projectDir, options) {
     console.log(`  🔍 Parsing dbt manifest: ${manifestPath}`);
 
     const tables = [];
+    const lineage = [];
     const domainsMap = new Map();
     const tableSplitKeyMap = new Map();
 
@@ -66,7 +67,6 @@ export async function importDbt(projectDir, options) {
         appearance: { type: 'table' },
         conceptual: { description: node.description || '' },
         columns,
-        lineage: { upstream: [] }
       };
 
       tables.push(tableEntry);
@@ -92,13 +92,12 @@ export async function importDbt(projectDir, options) {
     }
 
     // lineage
-    for (const [uniqueId, node] of Object.entries(allNodes)) {
+    for (const [, node] of Object.entries(allNodes)) {
       if (!['model', 'seed', 'snapshot', 'source'].includes(node.resource_type)) continue;
-      const tableEntry = tables.find(t => t.id === node.unique_id);
-      if (tableEntry && node.depends_on?.nodes) {
+      if (node.depends_on?.nodes) {
         for (const upstreamId of node.depends_on.nodes) {
           if (allNodes[upstreamId]) {
-            tableEntry.lineage.upstream.push(upstreamId);
+            lineage.push({ from: upstreamId, to: node.unique_id });
           }
         }
       }
@@ -122,11 +121,10 @@ export async function importDbt(projectDir, options) {
         const tableIds = new Set(splitTables.map(t => t.id));
         let internal = 0;
         let external = 0;
-        for (const table of splitTables) {
-          for (const upstreamId of table.lineage?.upstream || []) {
-            if (tableIds.has(upstreamId)) internal++;
-            else external++;
-          }
+        for (const edge of lineage) {
+          if (!tableIds.has(edge.to)) continue;
+          if (tableIds.has(edge.from)) internal++;
+          else external++;
         }
         const total = internal + external;
         const rate = total > 0 ? Math.round(internal / total * 100) : 100;
@@ -145,9 +143,12 @@ export async function importDbt(projectDir, options) {
             tables: d.tables.filter(tid => splitTables.some(t => t.id === tid))
           }));
 
+        const splitTableIds = new Set(splitTables.map(t => t.id));
+        const splitLineage = lineage.filter(e => splitTableIds.has(e.from) || splitTableIds.has(e.to));
         const outputModel = {
           tables: splitTables,
           relationships: [],
+          lineage: splitLineage,
           domains: splitDomains
         };
 
@@ -167,6 +168,7 @@ export async function importDbt(projectDir, options) {
       const outputModel = {
         tables,
         relationships: [],
+        lineage,
         domains: Array.from(domainsMap.values())
       };
 

package/src/sync-dbt.js

@@ -59,15 +59,6 @@ export async function syncDbt(projectDir, options) {
         }
       }
 
-      const lineageUpstream = [];
-      if (node.depends_on?.nodes) {
-        for (const upstreamId of node.depends_on.nodes) {
-          if (allNodes[upstreamId]) {
-            lineageUpstream.push(upstreamId);
-          }
-        }
-      }
-
       latestTablesMap.set(tableId, {
         id: tableId,
         name: node.name,
@@ -76,7 +67,6 @@ export async function syncDbt(projectDir, options) {
         appearance: { type: 'table' },
         conceptual: { description: node.description || '' },
         columns,
-        lineage: { upstream: lineageUpstream }
       });
     }
 
@@ -112,11 +102,24 @@ export async function syncDbt(projectDir, options) {
           physical_name: latest.physical_name,
           conceptual: latest.conceptual,
           columns: latest.columns,
-          lineage: latest.lineage
         };
       });
 
-      const updated = { ...existing, tables: newTables };
+      // Rebuild lineage from manifest for tables in this file
+      const fileTableIds = new Set(newTables.map(t => t.id));
+      const newLineage = [];
+      for (const [, node] of Object.entries(allNodes)) {
+        if (!fileTableIds.has(node.unique_id)) continue;
+        if (node.depends_on?.nodes) {
+          for (const upstreamId of node.depends_on.nodes) {
+            if (allNodes[upstreamId]) {
+              newLineage.push({ from: upstreamId, to: node.unique_id });
+            }
+          }
+        }
+      }
+
+      const updated = { ...existing, tables: newTables, lineage: newLineage };
       fs.writeFileSync(yamlPath, yaml.dump(updated), 'utf8');
       console.log(`  📄 Updated: ${yamlPath}`);
     }

package/src/templates/codegen-rules.md

@@ -8,11 +8,14 @@ Read this file alongside `.modscape/rules.md` (which defines the YAML schema) be
 
 ## 1. Dependency Order (DAG)
 
-Use `lineage.upstream` to determine build order. Always generate upstream models before downstream ones.
+Use the top-level `lineage` section to determine build order. Always generate upstream (`from`) models before downstream (`to`) ones.
 
 ```yaml
 lineage:
-  upstream: [stg_orders, stg_order_items]  # these must be generated first
+  - from: stg_orders       # must be generated first
+    to: fct_orders
+  - from: stg_order_items  # must be generated first
+    to: fct_orders
 ```
 
 In dbt this becomes `{{ ref('stg_orders') }}`. In SQLMesh, `MODEL (... grain [...])` with `@this_model` references. Apply the equivalent pattern for your target tool.
@@ -123,7 +126,7 @@ Common TODO patterns:
 
 ## 8. Physical Table Names
 
-When `physical_name` is set on a table, use it as the actual table name in DDL or config blocks. The `id` field is the logical reference name used in `ref()` calls and `lineage.upstream`.
+When `physical_name` is set on a table, use it as the actual table name in DDL or config blocks. The `id` field is the logical reference name used in `ref()` calls and the `lineage` section.
 
 ---
 

package/src/templates/rules.md

@@ -8,9 +8,9 @@
 ## QUICK REFERENCE (read this first)
 
 ```
-ROOT KEYS      domains | tables | relationships | annotations | layout
+ROOT KEYS      domains | tables | relationships | lineage | annotations | layout
 COORDINATES    ONLY in `layout`. NEVER inside tables or domains.
-LINEAGE        Use lineage.upstream (not relationships) for mart/aggregated tables.
+LINEAGE        Use top-level `lineage` section (not relationships, not table.lineage.upstream).
 parentId       Declare a table's domain membership inside layout, not inside domains.
 IDs            Every object (table, domain, annotation) needs a unique `id`.
 sampleData     First row = column IDs. At least 3 realistic data rows.
@@ -27,6 +27,7 @@ A valid `model.yaml` has exactly these top-level keys.
 domains:       # (array) visual containers — OPTIONAL but recommended
 tables:        # (array) entity definitions — REQUIRED
 relationships: # (array) ER cardinality edges — OPTIONAL
+lineage:       # (array) data lineage edges — OPTIONAL
 annotations:   # (array) sticky notes / callouts — OPTIONAL
 layout:        # (object) ALL coordinates — REQUIRED if any objects exist
 ```
@@ -140,17 +141,15 @@ relationships:
 
 ## 4. Data Lineage
 
-`lineage.upstream` declares which source tables a derived table is built from.
-This is rendered as animated arrows in **Lineage Mode**. It is separate from ER relationships.
+Top-level `lineage` section declares data flow between tables (which source tables feed which derived tables).
+This is rendered as dashed arrows in **Lineage Mode**. It is separate from ER relationships.
 
 ```yaml
-tables:
-  - id: mart_revenue
-    appearance: { type: mart }
-    lineage:
-      upstream:
-        - fct_orders    # list of source table IDs
-        - dim_dates
+lineage:
+  - from: fct_orders    # source table id
+    to: mart_revenue    # derived table id
+  - from: dim_dates
+    to: mart_revenue
 ```
 
 ### When to use lineage vs relationships
@@ -158,21 +157,21 @@ tables:
 | Situation | Use |
 |-----------|-----|
 | `dim_customers` → `fct_orders` (FK join) | `relationships` |
-| `fct_orders` + `dim_dates` → `mart_revenue` (aggregation) | `lineage.upstream` |
+| `fct_orders` + `dim_dates` → `mart_revenue` (aggregation) | `lineage` |
 
-**MUST** define `lineage.upstream` for every `mart` or aggregated table.
-**MUST NOT** define `lineage.upstream` for raw tables (`fact`, `dimension`, `hub`, `link`, `satellite`).
-**MUST NOT** add a `relationships` entry for a connection already expressed in `lineage.upstream`.
+**MUST** define `lineage` entries for every `mart` or aggregated table.
+**MUST NOT** define `lineage` entries for raw tables (`fact`, `dimension`, `hub`, `link`, `satellite`) as sources.
+**MUST NOT** add a `relationships` entry for a connection already expressed in `lineage`.
 
 #### Example: correct separation
 
 ```yaml
 # CORRECT
-tables:
-  - id: mart_revenue
-    appearance: { type: mart }
-    lineage:
-      upstream: [fct_orders, dim_dates]   # lineage only
+lineage:
+  - from: fct_orders
+    to: mart_revenue
+  - from: dim_dates
+    to: mart_revenue
 
 relationships:
   - from: { table: dim_customers, column: customer_key }
@@ -409,11 +408,9 @@ relationships:
 
 ```yaml
 # CORRECT
-tables:
-  - id: mart_revenue
-    appearance: { type: mart }
-    lineage:
-      upstream: [fct_orders]    # ✅ express lineage here
+lineage:
+  - from: fct_orders
+    to: mart_revenue    # ✅ express lineage in the top-level lineage section
 ```
 
 ---
@@ -663,10 +660,6 @@ tables:
     logical_name: "Executive Revenue Summary"
     physical_name: "mart_finance_monthly_revenue_agg"
     appearance: { type: mart, icon: "📈" }
-    lineage:                        # mart → use lineage, not relationships
-      upstream:
-        - fct_orders
-        - dim_customers
     implementation:
       materialization: table
       grain: [month_key]
@@ -684,6 +677,12 @@ tables:
       - ["2024-02", 15200.00]
       - ["2024-03", 18900.75]
 
+lineage:                            # data flow — separate from ER
+  - from: fct_orders
+    to: mart_monthly_revenue
+  - from: dim_customers
+    to: mart_monthly_revenue
+
 relationships:                      # ER only — not for lineage
   - from: { table: dim_customers, column: customer_key }
     to:   { table: fct_orders,    column: customer_key }

package/visualizer/package.json

@@ -1,13 +1,14 @@
 {
   "name": "visualizer",
   "private": true,
-  "version": "1.3.0",
+  "version": "2.0.0",
   "type": "module",
   "scripts": {
     "dev": "vite",
     "build": "tsc -b && vite build",
     "lint": "eslint .",
-    "preview": "vite preview"
+    "preview": "vite preview",
+    "test": "vitest run"
   },
   "dependencies": {
     "@codemirror/lang-yaml": "^6.1.2",
@@ -19,13 +20,16 @@
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "codemirror": "^6.0.2",
+    "cytoscape": "^3.33.1",
+    "cytoscape-dagre": "^2.5.0",
+    "cytoscape-dom-node": "^1.2.0",
+    "cytoscape-edgehandles": "^4.0.1",
     "dagre": "^0.8.5",
     "html-to-image": "^1.11.13",
     "js-yaml": "^4.1.1",
     "lucide-react": "^0.575.0",
     "react": "^19.2.0",
     "react-dom": "^19.2.0",
-    "reactflow": "^11.11.4",
     "tailwind-merge": "^3.5.0",
     "zustand": "^5.0.11"
   },
@@ -46,6 +50,7 @@
     "tailwindcss": "^3.4.19",
     "typescript": "~5.9.3",
     "typescript-eslint": "^8.48.0",
-    "vite": "^7.3.1"
+    "vite": "^7.3.1",
+    "vitest": "^4.1.0"
   }
 }