@lokascript/framework 2.0.0

package/package.json

@@ -0,0 +1,122 @@
+{
+  "name": "@lokascript/framework",
+  "version": "2.0.0",
+  "description": "Generic framework for building multilingual DSLs with semantic parsing and grammar transformation",
+  "type": "module",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    },
+    "./core": {
+      "types": "./dist/core/index.d.ts",
+      "import": "./dist/core/index.js"
+    },
+    "./core/types": {
+      "types": "./dist/core/types.d.ts",
+      "import": "./dist/core/types.js"
+    },
+    "./core/tokenization": {
+      "types": "./dist/core/tokenization/index.d.ts",
+      "import": "./dist/core/tokenization/index.js"
+    },
+    "./core/pattern-matching": {
+      "types": "./dist/core/pattern-matching/index.d.ts",
+      "import": "./dist/core/pattern-matching/index.js"
+    },
+    "./multilingual": {
+      "types": "./dist/multilingual/index.d.ts",
+      "import": "./dist/multilingual/index.js"
+    },
+    "./schema": {
+      "types": "./dist/schema/index.d.ts",
+      "import": "./dist/schema/index.js"
+    },
+    "./parsing": {
+      "types": "./dist/parsing/index.d.ts",
+      "import": "./dist/parsing/index.js"
+    },
+    "./generation": {
+      "types": "./dist/generation/index.d.ts",
+      "import": "./dist/generation/index.js"
+    },
+    "./grammar": {
+      "types": "./dist/grammar/index.d.ts",
+      "import": "./dist/grammar/index.js"
+    },
+    "./api": {
+      "types": "./dist/api/index.d.ts",
+      "import": "./dist/api/index.js"
+    },
+    "./testing": {
+      "types": "./dist/testing/index.d.ts",
+      "import": "./dist/testing/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsup && npm run build:types",
+    "build:types": "tsc --emitDeclarationOnly --outDir dist",
+    "dev": "tsup --watch",
+    "test": "vitest",
+    "test:run": "vitest run",
+    "test:watch": "vitest watch",
+    "test:check": "vitest run --reporter=dot 2>&1 | tail -5",
+    "test:coverage": "vitest run --coverage",
+    "test:unit": "vitest run src/core/**/*.test.ts src/generation/**/*.test.ts",
+    "test:integration": "vitest run src/__test__/**/*.test.ts src/api/**/*.test.ts",
+    "test:pattern-matcher": "vitest run src/core/pattern-matching",
+    "test:tokenizer": "vitest run src/core/tokenization",
+    "typecheck": "tsc --noEmit",
+    "create-domain": "npx tsx scripts/create-domain.ts"
+  },
+  "keywords": [
+    "dsl",
+    "multilingual",
+    "i18n",
+    "parser",
+    "semantic",
+    "pattern-matching",
+    "grammar-transformation",
+    "natural-language",
+    "framework",
+    "compiler",
+    "svo",
+    "sov",
+    "vso"
+  ],
+  "author": "LokaScript Contributors",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "@vitest/coverage-v8": "^4.0.17",
+    "tsup": "^8.0.0",
+    "typescript": "^5.0.0",
+    "vitest": "^4.0.0"
+  },
+  "files": [
+    "dist",
+    "src",
+    "LICENSE",
+    "README.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/codetalcott/hyperfixi.git",
+    "directory": "packages/framework"
+  },
+  "bugs": {
+    "url": "https://github.com/codetalcott/hyperfixi/issues"
+  },
+  "homepage": "https://github.com/codetalcott/hyperfixi/tree/main/packages/framework#readme",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "sideEffects": false
+}

package/src/__test__/fixtures/sql-dsl.ts

@@ -0,0 +1,232 @@
+/**
+ * SQL DSL Test Fixture
+ *
+ * Reusable SQL DSL configuration for integration tests.
+ * Based on examples/sql-dsl/simple-sql.ts
+ */
+
+import {
+  createMultilingualDSL,
+  defineCommand,
+  defineRole,
+  BaseTokenizer,
+  getDefaultExtractors,
+  type SemanticNode,
+  type MultilingualDSL,
+} from '../../index';
+
+// =============================================================================
+// Command Schemas
+// =============================================================================
+
+export const selectCommand = defineCommand({
+  action: 'select',
+  description: 'Retrieve data from a table',
+  category: 'query',
+  primaryRole: 'columns',
+  roles: [
+    defineRole({
+      role: 'columns',
+      description: 'Columns to select',
+      required: true,
+      expectedTypes: ['expression'],
+      svoPosition: 2,
+      sovPosition: 1,
+    }),
+    defineRole({
+      role: 'source',
+      description: 'Table to select from',
+      required: true,
+      expectedTypes: ['expression'],
+      svoPosition: 1,
+      sovPosition: 2,
+      markerOverride: { en: 'from', es: 'de' },
+    }),
+    defineRole({
+      role: 'condition',
+      description: 'WHERE clause condition',
+      required: false,
+      expectedTypes: ['expression'],
+      svoPosition: 0,
+      sovPosition: 0,
+      markerOverride: { en: 'where', es: 'donde' },
+    }),
+  ],
+});
+
+// =============================================================================
+// Tokenizers
+// =============================================================================
+
+class EnglishSQLTokenizer extends BaseTokenizer {
+  readonly language = 'en';
+  readonly direction = 'ltr' as const;
+
+  constructor() {
+    super();
+    this.registerExtractors(getDefaultExtractors());
+  }
+
+  classifyToken(token: string): 'keyword' | 'identifier' | 'literal' | 'operator' {
+    const keywords = ['select', 'from', 'where'];
+    if (keywords.includes(token.toLowerCase())) {
+      return 'keyword';
+    }
+    if (/^\d+$/.test(token)) {
+      return 'literal';
+    }
+    if (['>', '<', '=', '>=', '<=', '!='].includes(token)) {
+      return 'operator';
+    }
+    return 'identifier';
+  }
+}
+
+class SpanishSQLTokenizer extends BaseTokenizer {
+  readonly language = 'es';
+  readonly direction = 'ltr' as const;
+
+  constructor() {
+    super();
+    this.registerExtractors(getDefaultExtractors());
+  }
+
+  classifyToken(token: string): 'keyword' | 'identifier' | 'literal' | 'operator' {
+    const keywords = ['seleccionar', 'de', 'donde'];
+    if (keywords.includes(token.toLowerCase())) {
+      return 'keyword';
+    }
+    if (/^\d+$/.test(token)) {
+      return 'literal';
+    }
+    if (['>', '<', '=', '>=', '<=', '!='].includes(token)) {
+      return 'operator';
+    }
+    return 'identifier';
+  }
+}
+
+// =============================================================================
+// Language Profiles
+// =============================================================================
+
+const englishProfile = {
+  code: 'en',
+  wordOrder: 'SVO' as const,
+  keywords: {
+    select: { primary: 'select' },
+  },
+  roleMarkers: {
+    source: { primary: 'from', position: 'before' as const },
+    condition: { primary: 'where', position: 'before' as const },
+  },
+};
+
+const spanishProfile = {
+  code: 'es',
+  wordOrder: 'SVO' as const,
+  keywords: {
+    select: { primary: 'seleccionar' },
+  },
+  roleMarkers: {
+    source: { primary: 'de', position: 'before' as const },
+    condition: { primary: 'donde', position: 'before' as const },
+  },
+};
+
+// =============================================================================
+// Code Generator
+// =============================================================================
+
+function generateSQL(node: SemanticNode): string {
+  if (node.action === 'select') {
+    const columns = node.roles.get('columns');
+    const source = node.roles.get('source');
+    const condition = node.roles.get('condition');
+
+    const columnsValue = columns ? ('raw' in columns ? columns.raw : columns.value) : '*';
+    const sourceValue = source ? ('raw' in source ? source.raw : source.value) : 'table';
+
+    let sql = `SELECT ${columnsValue} FROM ${sourceValue}`;
+
+    if (condition) {
+      const conditionValue = 'raw' in condition ? condition.raw : condition.value;
+      sql += ` WHERE ${conditionValue}`;
+    }
+
+    return sql;
+  }
+
+  throw new Error(`Unknown SQL command: ${node.action}`);
+}
+
+// =============================================================================
+// SQL DSL Instance
+// =============================================================================
+
+export function createSQLDSL(): MultilingualDSL {
+  return createMultilingualDSL({
+    name: 'SQL DSL Test Fixture',
+    schemas: [selectCommand],
+    languages: [
+      {
+        code: 'en',
+        name: 'English',
+        nativeName: 'English',
+        tokenizer: new EnglishSQLTokenizer(),
+        patternProfile: englishProfile,
+      },
+      {
+        code: 'es',
+        name: 'Spanish',
+        nativeName: 'Español',
+        tokenizer: new SpanishSQLTokenizer(),
+        patternProfile: spanishProfile,
+      },
+    ],
+    codeGenerator: {
+      generate: generateSQL,
+    },
+  });
+}
+
+// =============================================================================
+// Test Data
+// =============================================================================
+
+export const testQueries = {
+  english: {
+    simple: 'select name from users',
+    withWhere: 'select name from users where age > 18',
+    multipleColumns: 'select name, email from users',
+  },
+  spanish: {
+    simple: 'seleccionar nombre de usuarios',
+    withWhere: 'seleccionar nombre de usuarios donde edad > 18',
+    multipleColumns: 'seleccionar nombre, email de usuarios',
+  },
+};
+
+export const expectedSQL = {
+  simple: 'SELECT name FROM users',
+  withWhere: 'SELECT name FROM users WHERE age > 18',
+  multipleColumns: 'SELECT name, email FROM users',
+};
+
+export const expectedNodes = {
+  simple: {
+    action: 'select',
+    roles: {
+      columns: 'name',
+      source: 'users',
+    },
+  },
+  withWhere: {
+    action: 'select',
+    roles: {
+      columns: 'name',
+      source: 'users',
+      condition: 'age > 18',
+    },
+  },
+};

package/src/__test__/sql-integration.test.ts

@@ -0,0 +1,189 @@
+/**
+ * SQL DSL Integration Tests
+ *
+ * End-to-end tests using the SQL DSL fixture to validate the full pipeline:
+ * Tokenization → Pattern Matching → Semantic Parsing → Code Generation
+ */
+
+import { describe, it, expect } from 'vitest';
+import { createSQLDSL, testQueries, expectedSQL, expectedNodes } from './fixtures/sql-dsl';
+
+describe('SQL DSL Integration', () => {
+  let sqlDSL: ReturnType<typeof createSQLDSL>;
+
+  beforeEach(() => {
+    sqlDSL = createSQLDSL();
+  });
+
+  describe('English Queries', () => {
+    it('should parse simple SELECT query', () => {
+      const result = sqlDSL.parse(testQueries.english.simple, 'en');
+
+      expect(result.kind).toBe('command');
+      expect(result.action).toBe('select');
+      expect(result.roles.has('columns')).toBe(true);
+      expect(result.roles.has('source')).toBe(true);
+
+      const columns = result.roles.get('columns');
+      const source = result.roles.get('source');
+
+      if (columns && 'raw' in columns) {
+        expect(columns.raw).toBe('name');
+      }
+      if (source && 'raw' in source) {
+        expect(source.raw).toBe('users');
+      }
+    });
+
+    it('should parse SELECT with WHERE clause', () => {
+      const result = sqlDSL.parse(testQueries.english.withWhere, 'en');
+
+      expect(result.action).toBe('select');
+      expect(result.roles.has('columns')).toBe(true);
+      expect(result.roles.has('source')).toBe(true);
+      expect(result.roles.has('condition')).toBe(true);
+
+      const condition = result.roles.get('condition');
+      expect(condition).toBeDefined();
+      if (condition && 'raw' in condition) {
+        // The condition role captures the expression after WHERE
+        expect(condition.raw).toBeTruthy();
+      }
+    });
+  });
+
+  describe('Spanish Queries', () => {
+    it('should parse Spanish SELECT query', () => {
+      const result = sqlDSL.parse(testQueries.spanish.simple, 'es');
+
+      expect(result.action).toBe('select');
+      expect(result.roles.has('columns')).toBe(true);
+      expect(result.roles.has('source')).toBe(true);
+
+      const columns = result.roles.get('columns');
+      const source = result.roles.get('source');
+
+      if (columns && 'raw' in columns) {
+        expect(columns.raw).toBe('nombre');
+      }
+      if (source && 'raw' in source) {
+        expect(source.raw).toBe('usuarios');
+      }
+    });
+
+    it('should parse Spanish SELECT with WHERE clause', () => {
+      const result = sqlDSL.parse(testQueries.spanish.withWhere, 'es');
+
+      expect(result.action).toBe('select');
+      expect(result.roles.has('condition')).toBe(true);
+    });
+  });
+
+  describe('Code Generation', () => {
+    it('should compile English query to SQL', () => {
+      const result = sqlDSL.compile(testQueries.english.simple, 'en');
+
+      expect(result.ok).toBe(true);
+      expect(result.code).toBe(expectedSQL.simple);
+    });
+
+    it('should compile query with WHERE clause', () => {
+      const result = sqlDSL.compile(testQueries.english.withWhere, 'en');
+
+      expect(result.ok).toBe(true);
+      // Should contain all parts
+      expect(result.code).toContain('SELECT');
+      expect(result.code).toContain('FROM');
+      expect(result.code).toContain('WHERE');
+    });
+
+    it('should preserve identifiers from source language', () => {
+      const enResult = sqlDSL.compile(testQueries.english.simple, 'en');
+      const esResult = sqlDSL.compile(testQueries.spanish.simple, 'es');
+
+      // English uses English identifiers
+      expect(enResult.code).toContain('name');
+      expect(enResult.code).toContain('users');
+
+      // Spanish uses Spanish identifiers
+      expect(esResult.code).toContain('nombre');
+      expect(esResult.code).toContain('usuarios');
+
+      // But both have SQL structure
+      expect(enResult.code).toContain('SELECT');
+      expect(esResult.code).toContain('SELECT');
+    });
+  });
+
+  describe('Validation', () => {
+    it('should validate correct queries', () => {
+      const result = sqlDSL.validate(testQueries.english.simple, 'en');
+
+      expect(result.valid).toBe(true);
+      expect(result.node).toBeDefined();
+      expect(result.errors).toBeUndefined();
+    });
+
+    it('should handle invalid queries gracefully', () => {
+      // Query with unknown keyword
+      const result = sqlDSL.validate('invalid query syntax', 'en');
+
+      expect(result.valid).toBe(false);
+      expect(result.errors).toBeDefined();
+      expect(result.errors?.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('Multi-language Support', () => {
+    it('should support both English and Spanish', () => {
+      const languages = sqlDSL.getSupportedLanguages();
+
+      expect(languages).toContain('en');
+      expect(languages).toContain('es');
+      expect(languages).toHaveLength(2);
+    });
+
+    it('should reject unsupported language', () => {
+      expect(() => {
+        sqlDSL.parse('select name from users', 'fr');
+      }).toThrow();
+    });
+  });
+
+  describe('Semantic Equivalence', () => {
+    it('should parse English and Spanish to equivalent semantic structures', () => {
+      const enNode = sqlDSL.parse(testQueries.english.simple, 'en');
+      const esNode = sqlDSL.parse(testQueries.spanish.simple, 'es');
+
+      expect(enNode.action).toBe(esNode.action);
+      expect(enNode.roles.size).toBe(esNode.roles.size);
+      expect(enNode.roles.has('columns')).toBe(esNode.roles.has('columns'));
+      expect(enNode.roles.has('source')).toBe(esNode.roles.has('source'));
+    });
+  });
+
+  describe('Error Handling', () => {
+    it('should handle empty input', () => {
+      expect(() => {
+        sqlDSL.parse('', 'en');
+      }).toThrow();
+    });
+
+    it('should handle whitespace-only input', () => {
+      expect(() => {
+        sqlDSL.parse('   ', 'en');
+      }).toThrow();
+    });
+
+    it('should provide helpful error messages', () => {
+      try {
+        sqlDSL.parse('unknown command', 'en');
+        // Should have thrown
+        expect(true).toBe(false);
+      } catch (error) {
+        expect(error).toBeDefined();
+        expect(error instanceof Error).toBe(true);
+      }
+    });
+  });
+});