@mmapp/react-compiler 0.1.0-alpha.1

package/src/envelope.ts

@@ -0,0 +1,315 @@
+/**
+ * Envelope Builder — walks a project directory, compiles all .workflow.tsx files,
+ * and produces a content-addressed SHA-256 deployment bundle.
+ *
+ * This is the high-level API that combines:
+ *   - File discovery (fs-tree)
+ *   - Compilation (babel plugin)
+ *   - Source binding extraction
+ *   - Content-addressed envelope creation
+ *
+ * Usage:
+ *   import { buildEnvelope } from '@mindmatrix/react-compiler';
+ *   const envelope = await buildEnvelope('./src/workflows', { slug: 'my-app' });
+ *   // envelope.manifest.hash === 'a1b2c3...' (SHA-256)
+ */
+
+import { createHash } from 'crypto';
+import { readFileSync } from 'fs';
+import { resolve, relative, basename } from 'path';
+import { glob } from 'glob';
+import { transformSync } from '@babel/core';
+import babelPlugin from './babel';
+import { generateFsTree } from './envelope/fs-tree';
+import {
+  createSourceEnvelope,
+  type SourceEnvelope,
+  type SourceBinding,
+} from './envelope/source-envelope';
+import type { IRWorkflowDefinition } from '@mindmatrix/player-core';
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface EnvelopeManifest {
+  /** Content-addressed SHA-256 hash of the entire envelope. */
+  hash: string;
+  /** Files included in the envelope (relative paths). */
+  files: EnvelopeFile[];
+  /** Blueprint version string. */
+  version: string;
+  /** ISO 8601 timestamp. */
+  timestamp: string;
+  /** Total compiled definitions. */
+  definitionCount: number;
+  /** Total compilation errors. */
+  errorCount: number;
+}
+
+export interface EnvelopeFile {
+  /** Relative path from project root. */
+  path: string;
+  /** SHA-256 hash of file content. */
+  hash: string;
+  /** File size in bytes. */
+  size: number;
+  /** Workflow slug (if this file produced a definition). */
+  slug?: string;
+  /** Whether compilation succeeded for this file. */
+  compiled: boolean;
+}
+
+export interface BuildEnvelopeOptions {
+  /** Blueprint slug. Default: basename of projectDir */
+  slug?: string;
+  /** Blueprint version. Default: '0.1.0' */
+  version?: string;
+  /** Compilation mode. Default: 'infer' */
+  mode?: 'strict' | 'infer';
+  /** Glob patterns for files to include. Default: discovers all workflow-related files */
+  include?: string[];
+  /** Environment tag. Default: undefined */
+  environment?: string;
+}
+
+export interface BuildEnvelopeResult {
+  /** The complete source envelope. */
+  envelope: SourceEnvelope;
+  /** Content-addressed manifest for the bundle. */
+  manifest: EnvelopeManifest;
+  /** Individual file compilation results. */
+  files: EnvelopeFile[];
+  /** Definitions that compiled successfully. */
+  definitions: IRWorkflowDefinition[];
+  /** Compilation errors. */
+  errors: Array<{ file: string; message: string; line?: number }>;
+}
+
+// =============================================================================
+// Builder
+// =============================================================================
+
+/**
+ * Walk a project directory, compile all workflow files, and produce a
+ * content-addressed deployment bundle.
+ */
+export async function buildEnvelope(
+  projectDir: string,
+  options: BuildEnvelopeOptions = {},
+): Promise<BuildEnvelopeResult> {
+  const root = resolve(projectDir);
+  const slug = options.slug ?? basename(root);
+  const version = options.version ?? '0.1.0';
+  const mode = options.mode ?? 'infer';
+
+  // Discover files
+  const patterns = options.include ?? [
+    `${root}/**/*.workflow.{tsx,ts,jsx,js}`,
+    `${root}/**/models/**/*.{ts,tsx}`,
+    `${root}/**/*.server.{ts,tsx,js,jsx}`,
+  ];
+
+  const discoveredFiles: string[] = [];
+  for (const pattern of patterns) {
+    const matches = await glob(pattern, { absolute: true });
+    discoveredFiles.push(...matches);
+  }
+  const uniqueFiles = [...new Set(discoveredFiles)];
+
+  // Also discover all source files for the fs-tree
+  const allSourceFiles = await glob(`${root}/**/*.{ts,tsx,js,jsx,css,json}`, {
+    absolute: true,
+    ignore: ['**/node_modules/**', '**/dist/**', '**/.git/**'],
+  });
+
+  // Generate fs-tree
+  const fsTree = generateFsTree(allSourceFiles, root);
+
+  // Compile each file
+  const definitions: IRWorkflowDefinition[] = [];
+  const envelopeFiles: EnvelopeFile[] = [];
+  const errors: Array<{ file: string; message: string; line?: number }> = [];
+  const sourceBindings: SourceBinding[] = [];
+
+  for (const file of uniqueFiles) {
+    const relativePath = relative(root, file);
+    let fileContent: Buffer;
+    try {
+      fileContent = readFileSync(file) as unknown as Buffer;
+    } catch {
+      errors.push({ file: relativePath, message: 'Could not read file' });
+      continue;
+    }
+
+    const fileHash = createHash('sha256').update(fileContent).digest('hex');
+    const code = fileContent.toString('utf-8');
+
+    try {
+      // Determine parser plugins based on file type
+      const hasTsx = /\.(tsx|jsx)$/.test(file);
+      const parserPlugins: string[] = ['typescript'];
+      if (hasTsx) parserPlugins.push('jsx');
+
+      const result = transformSync(code, {
+        filename: file,
+        plugins: [[babelPlugin, { mode }]],
+        parserOpts: { plugins: parserPlugins as any, attachComment: true },
+        sourceMaps: true,
+      });
+
+      const ir = (result as any)?.metadata?.mindmatrixIR as IRWorkflowDefinition | undefined;
+
+      if (ir) {
+        definitions.push(ir);
+
+        // Extract source bindings from IR
+        const bindings = extractSourceBindings(ir, relativePath);
+        sourceBindings.push(...bindings);
+
+        // Check for compilation errors in IR metadata
+        const irErrors = (ir.metadata as any)?.errors;
+        if (irErrors?.length) {
+          for (const e of irErrors) {
+            errors.push({ file: relativePath, message: e.message, line: e.line });
+          }
+        }
+
+        envelopeFiles.push({
+          path: relativePath,
+          hash: fileHash,
+          size: fileContent.length,
+          slug: ir.slug,
+          compiled: true,
+        });
+      } else {
+        envelopeFiles.push({
+          path: relativePath,
+          hash: fileHash,
+          size: fileContent.length,
+          compiled: false,
+        });
+      }
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : String(error);
+      errors.push({ file: relativePath, message: msg });
+      envelopeFiles.push({
+        path: relativePath,
+        hash: fileHash,
+        size: fileContent.length,
+        compiled: false,
+      });
+    }
+  }
+
+  // Create source envelope
+  const envelope = createSourceEnvelope({
+    blueprintSlug: slug,
+    blueprintVersion: version,
+    definitions,
+    fsTree,
+    sourceBindings,
+    environment: options.environment,
+    mode,
+  });
+
+  // Compute manifest hash (SHA-256 of the full envelope JSON)
+  const envelopeJson = JSON.stringify(envelope);
+  const manifestHash = createHash('sha256').update(envelopeJson).digest('hex');
+
+  const manifest: EnvelopeManifest = {
+    hash: manifestHash,
+    files: envelopeFiles,
+    version,
+    timestamp: envelope.createdAt,
+    definitionCount: definitions.length,
+    errorCount: errors.length,
+  };
+
+  return {
+    envelope,
+    manifest,
+    files: envelopeFiles,
+    definitions,
+    errors,
+  };
+}
+
+// =============================================================================
+// Source Binding Extraction
+// =============================================================================
+
+/**
+ * Extract source bindings from a compiled IR definition.
+ * Maps IR elements (fields, states, transitions) back to source file.
+ */
+function extractSourceBindings(
+  ir: IRWorkflowDefinition,
+  filePath: string,
+): SourceBinding[] {
+  const bindings: SourceBinding[] = [];
+  const slug = ir.slug ?? '';
+
+  // Fields
+  for (const field of ir.fields) {
+    const source = (field as any).__source;
+    bindings.push({
+      kind: 'field',
+      name: field.name,
+      workflowSlug: slug,
+      file: filePath,
+      startLine: source?.line ?? 0,
+      startColumn: source?.column ?? 0,
+      endLine: source?.endLine,
+      endColumn: source?.endColumn,
+    });
+  }
+
+  // States
+  for (const state of ir.states) {
+    const source = (state as any).__source;
+    bindings.push({
+      kind: 'state',
+      name: state.name,
+      workflowSlug: slug,
+      file: filePath,
+      startLine: source?.line ?? 0,
+      startColumn: source?.column ?? 0,
+      endLine: source?.endLine,
+      endColumn: source?.endColumn,
+    });
+  }
+
+  // Transitions
+  for (const transition of ir.transitions) {
+    const source = (transition as any).__source;
+    bindings.push({
+      kind: 'transition',
+      name: transition.name,
+      workflowSlug: slug,
+      file: filePath,
+      startLine: source?.line ?? 0,
+      startColumn: source?.column ?? 0,
+      endLine: source?.endLine,
+      endColumn: source?.endColumn,
+    });
+  }
+
+  // Grammar islands
+  if (ir.extensions) {
+    for (const [tag, islands] of Object.entries(ir.extensions)) {
+      for (const island of islands) {
+        bindings.push({
+          kind: 'grammar-island',
+          name: `${tag}:${island.contextTag}`,
+          workflowSlug: slug,
+          file: filePath,
+          startLine: (island as any).__source?.line ?? 0,
+          startColumn: (island as any).__source?.column ?? 0,
+        });
+      }
+    }
+  }
+
+  return bindings;
+}

package/src/incremental-compiler.ts

@@ -0,0 +1,401 @@
+/**
+ * Incremental Compiler — content-hash based incremental compilation.
+ *
+ * Maintains a cache of per-file compilation results (IR + errors) and
+ * only recompiles files whose content hash has changed. Merges changed
+ * IR into the cached full project IR.
+ *
+ * Performance target: <200ms for a single-file change in a 30-file project.
+ *
+ * Architecture:
+ *   1. Hash each file's content with djb2 (fast, no crypto overhead)
+ *   2. Compare against cached hashes to find dirty files
+ *   3. Recompile only dirty files
+ *   4. Merge fresh IR with cached IR for unchanged files
+ *   5. Rebuild composed result
+ *
+ * Also tracks dependency edges so that when a model file changes,
+ * workflow files that import from it are marked dirty too.
+ */
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface FileHash {
+  /** djb2 hash of the file content. */
+  hash: string;
+  /** Timestamp of last compilation. */
+  compiledAt: number;
+  /** File size in characters. */
+  size: number;
+}
+
+export interface IncrementalStats {
+  /** Total files in cache. */
+  totalCached: number;
+  /** Files recompiled in last compilation. */
+  lastRecompiled: number;
+  /** Files served from cache in last compilation. */
+  lastCacheHits: number;
+  /** Total compilation time in ms for last compile. */
+  lastCompileMs: number;
+  /** Average per-file compilation time in ms. */
+  avgPerFileMs: number;
+  /** Cache hit rate (0-1). */
+  cacheHitRate: number;
+}
+
+export interface DependencyEdge {
+  /** The file that imports. */
+  from: string;
+  /** The file being imported. */
+  to: string;
+}
+
+export interface DirtyFileSet {
+  /** Files whose content hash changed. */
+  contentChanged: string[];
+  /** Files dirty due to dependency on a changed file. */
+  dependencyDirty: string[];
+  /** Files removed since last compilation. */
+  removed: string[];
+  /** Files added since last compilation. */
+  added: string[];
+}
+
+// =============================================================================
+// Content Hashing
+// =============================================================================
+
+/**
+ * Fast content hash using djb2 algorithm.
+ * Returns a short base-36 string suitable for cache invalidation.
+ */
+export function hashContent(content: string): string {
+  let hash = 5381;
+  for (let i = 0; i < content.length; i++) {
+    hash = ((hash << 5) + hash) + content.charCodeAt(i);
+    hash |= 0; // Convert to 32-bit integer
+  }
+  return (hash >>> 0).toString(36);
+}
+
+// =============================================================================
+// Import Dependency Graph
+// =============================================================================
+
+/**
+ * Extracts import paths from a source file using regex.
+ * Returns relative import targets (e.g., ['../models/user', './components/Map']).
+ */
+export function extractImports(source: string): string[] {
+  const imports: string[] = [];
+  const regex = /import\s+.*?from\s+['"](\.[^'"]+)['"]/g;
+  let match;
+  while ((match = regex.exec(source)) !== null) {
+    imports.push(match[1]);
+  }
+  return imports;
+}
+
+/**
+ * Resolves a relative import to an absolute filename within the project.
+ */
+export function resolveImport(
+  fromFile: string,
+  importPath: string,
+  availableFiles: string[],
+): string | null {
+  const parts = fromFile.split('/');
+  parts.pop();
+  const dir = parts.join('/');
+
+  const segments = (dir ? dir + '/' + importPath : importPath).split('/');
+  const resolved: string[] = [];
+  for (const seg of segments) {
+    if (seg === '.') continue;
+    if (seg === '..') { resolved.pop(); continue; }
+    resolved.push(seg);
+  }
+  const basePath = resolved.join('/');
+
+  const extensions = ['', '.ts', '.tsx', '.js', '.jsx'];
+  for (const ext of extensions) {
+    const candidate = basePath + ext;
+    if (availableFiles.includes(candidate)) {
+      return candidate;
+    }
+  }
+
+  for (const ext of ['.ts', '.tsx', '.js', '.jsx']) {
+    const candidate = basePath + '/index' + ext;
+    if (availableFiles.includes(candidate)) {
+      return candidate;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Builds a dependency graph from a set of files.
+ * Returns edges and a reverse dependency map (file → dependents).
+ */
+export function buildDependencyGraph(
+  files: Record<string, string>,
+): {
+  edges: DependencyEdge[];
+  dependents: Map<string, Set<string>>;
+  dependencies: Map<string, Set<string>>;
+} {
+  const availableFiles = Object.keys(files);
+  const edges: DependencyEdge[] = [];
+  const dependents = new Map<string, Set<string>>();
+  const dependencies = new Map<string, Set<string>>();
+
+  for (const filename of availableFiles) {
+    dependents.set(filename, new Set());
+    dependencies.set(filename, new Set());
+  }
+
+  for (const [filename, source] of Object.entries(files)) {
+    const imports = extractImports(source);
+    for (const imp of imports) {
+      const resolved = resolveImport(filename, imp, availableFiles);
+      if (resolved && resolved !== filename) {
+        edges.push({ from: filename, to: resolved });
+        dependencies.get(filename)!.add(resolved);
+        if (!dependents.has(resolved)) dependents.set(resolved, new Set());
+        dependents.get(resolved)!.add(filename);
+      }
+    }
+  }
+
+  return { edges, dependents, dependencies };
+}
+
+/**
+ * Given a set of directly changed files, computes the full set of dirty files
+ * by following the dependency graph (transitive dependents).
+ */
+export function computeTransitiveDirtySet(
+  directlyChanged: Set<string>,
+  dependents: Map<string, Set<string>>,
+): Set<string> {
+  const dirty = new Set(directlyChanged);
+  const queue = [...directlyChanged];
+
+  while (queue.length > 0) {
+    const file = queue.shift()!;
+    const deps = dependents.get(file);
+    if (!deps) continue;
+    for (const dep of deps) {
+      if (!dirty.has(dep)) {
+        dirty.add(dep);
+        queue.push(dep);
+      }
+    }
+  }
+
+  return dirty;
+}
+
+// =============================================================================
+// Topological Sort
+// =============================================================================
+
+/**
+ * Returns files in compilation order (dependencies first).
+ * Uses Kahn's algorithm for topological sorting.
+ */
+export function topologicalSort(
+  files: string[],
+  dependencies: Map<string, Set<string>>,
+): string[] {
+  const inDegree = new Map<string, number>();
+  const dependentMap = new Map<string, Set<string>>();
+
+  for (const f of files) {
+    inDegree.set(f, 0);
+    dependentMap.set(f, new Set());
+  }
+
+  for (const f of files) {
+    const deps = dependencies.get(f);
+    if (!deps) continue;
+    for (const dep of deps) {
+      if (files.includes(dep)) {
+        inDegree.set(f, (inDegree.get(f) || 0) + 1);
+        if (!dependentMap.has(dep)) dependentMap.set(dep, new Set());
+        dependentMap.get(dep)!.add(f);
+      }
+    }
+  }
+
+  const queue: string[] = [];
+  for (const [f, degree] of inDegree) {
+    if (degree === 0) queue.push(f);
+  }
+
+  const order: string[] = [];
+  while (queue.length > 0) {
+    const f = queue.shift()!;
+    order.push(f);
+    for (const dependent of dependentMap.get(f) || []) {
+      const newDegree = (inDegree.get(dependent) || 1) - 1;
+      inDegree.set(dependent, newDegree);
+      if (newDegree === 0) queue.push(dependent);
+    }
+  }
+
+  // Append any remaining files (cycle detected)
+  for (const f of files) {
+    if (!order.includes(f)) {
+      order.push(f);
+    }
+  }
+
+  return order;
+}
+
+// =============================================================================
+// Incremental Cache Manager
+// =============================================================================
+
+/**
+ * Manages file hashes and determines which files need recompilation.
+ */
+export class IncrementalCache<T> {
+  private hashes = new Map<string, FileHash>();
+  private results = new Map<string, T>();
+  private lastStats: IncrementalStats = {
+    totalCached: 0,
+    lastRecompiled: 0,
+    lastCacheHits: 0,
+    lastCompileMs: 0,
+    avgPerFileMs: 0,
+    cacheHitRate: 0,
+  };
+
+  /**
+   * Determines which files are dirty given current file contents.
+   */
+  detectDirtyFiles(
+    files: Record<string, string>,
+    dependents?: Map<string, Set<string>>,
+  ): DirtyFileSet {
+    const currentFiles = new Set(Object.keys(files));
+    const contentChanged: string[] = [];
+    const removed: string[] = [];
+    const added: string[] = [];
+
+    // Check for changed or new files
+    for (const [filename, source] of Object.entries(files)) {
+      const hash = hashContent(source);
+      const cached = this.hashes.get(filename);
+
+      if (!cached) {
+        added.push(filename);
+      } else if (cached.hash !== hash) {
+        contentChanged.push(filename);
+      }
+    }
+
+    // Check for removed files
+    for (const cached of this.hashes.keys()) {
+      if (!currentFiles.has(cached)) {
+        removed.push(cached);
+      }
+    }
+
+    // Compute dependency-dirty files
+    const directlyDirty = new Set([...contentChanged, ...added]);
+    let dependencyDirty: string[] = [];
+
+    if (dependents && directlyDirty.size > 0) {
+      const allDirty = computeTransitiveDirtySet(directlyDirty, dependents);
+      dependencyDirty = [...allDirty].filter(f => !directlyDirty.has(f));
+    }
+
+    return { contentChanged, dependencyDirty, removed, added };
+  }
+
+  /**
+   * Updates the cache with a compilation result.
+   */
+  set(filename: string, source: string, result: T): void {
+    this.hashes.set(filename, {
+      hash: hashContent(source),
+      compiledAt: Date.now(),
+      size: source.length,
+    });
+    this.results.set(filename, result);
+  }
+
+  /**
+   * Gets a cached result.
+   */
+  get(filename: string): T | undefined {
+    return this.results.get(filename);
+  }
+
+  /**
+   * Checks if a file is in cache and not dirty.
+   */
+  has(filename: string): boolean {
+    return this.results.has(filename);
+  }
+
+  /**
+   * Removes a file from cache.
+   */
+  delete(filename: string): void {
+    this.hashes.delete(filename);
+    this.results.delete(filename);
+  }
+
+  /**
+   * Clears all cached data.
+   */
+  clear(): void {
+    this.hashes.clear();
+    this.results.clear();
+  }
+
+  /**
+   * Updates stats after a compilation run.
+   */
+  updateStats(recompiled: number, cacheHits: number, compileMs: number): void {
+    const total = recompiled + cacheHits;
+    this.lastStats = {
+      totalCached: this.results.size,
+      lastRecompiled: recompiled,
+      lastCacheHits: cacheHits,
+      lastCompileMs: compileMs,
+      avgPerFileMs: recompiled > 0 ? compileMs / recompiled : 0,
+      cacheHitRate: total > 0 ? cacheHits / total : 0,
+    };
+  }
+
+  /**
+   * Gets compilation statistics.
+   */
+  getStats(): IncrementalStats {
+    return { ...this.lastStats };
+  }
+
+  /**
+   * Gets all cached filenames.
+   */
+  getCachedFiles(): string[] {
+    return [...this.results.keys()];
+  }
+
+  /**
+   * Gets all cached results.
+   */
+  getAllResults(): Map<string, T> {
+    return new Map(this.results);
+  }
+}