@lokascript/framework 2.8.0 → 2.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lokascript/framework",
-  "version": "2.8.0",
+  "version": "2.9.1",
   "description": "Generic framework for building multilingual DSLs with semantic parsing and grammar transformation",
   "type": "module",
   "main": "dist/index.cjs",
@@ -96,7 +96,7 @@
   "author": "LokaScript Contributors",
   "license": "MIT",
   "dependencies": {
-    "@lokascript/intent": "^2.8.0"
+    "@lokascript/intent": "^2.9.1"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",
@@ -109,7 +109,8 @@
     "dist",
     "src",
     "LICENSE",
-    "README.md"
+    "README.md",
+    "CHANGELOG.md"
   ],
   "repository": {
     "type": "git",

package/src/api/create-dsl.test.ts

@@ -279,6 +279,17 @@ describe('MultilingualDSL: Explicit Syntax Support', () => {
       const result = dsl.translate('[select columns:name source:users]', 'en', 'ja');
       expect(result).toBe('[select columns:name source:users]');
     });
+
+    // The test DSL configures no `grammarProfile`, which is legal for
+    // parse/validate/compile but not for translate. The error must name the
+    // field to set rather than leaving the caller with "No profile found".
+    it('throws an error naming grammarProfile when the profile is missing', () => {
+      const dsl = createTestDSL();
+      expect(() => dsl.translate('select name from users', 'en', 'ja')).toThrow(/grammarProfile/);
+      expect(() => dsl.translate('select name from users', 'en', 'ja')).toThrow(
+        /parse\/validate\/compile do not need it/
+      );
+    });
   });
 
   // ---------------------------------------------------------------------------

package/src/api/create-dsl.ts

@@ -6,6 +6,7 @@ import type { SemanticNode, LanguageTokenizer, LanguagePattern } from '../core/t
 import type { CommandSchema } from '../schema';
 import { PatternMatcher, type PatternMatcherProfile } from '../core/pattern-matching';
 import { generatePattern, type PatternGenLanguageProfile } from '../generation/pattern-generator';
+import { createDomainRenderer, type DomainRenderFn } from '../generation/renderer';
 import { GrammarTransformer, type LanguageProfile as GrammarProfile } from '../grammar';
 import {
   type Dictionary,
@@ -31,7 +32,12 @@ export interface LanguageConfig {
   readonly tokenizer: LanguageTokenizer;
   /** Language profile for pattern generation */
   readonly patternProfile: PatternGenLanguageProfile;
-  /** Language profile for grammar transformation (optional) */
+  /**
+   * Language profile for grammar transformation.
+   *
+   * Optional for `parse()`, `validate()` and `compile()`, but **required for
+   * `translate()`** — omitting it makes `translate()` throw for this language.
+   */
   readonly grammarProfile?: GrammarProfile;
 }
 
@@ -42,6 +48,77 @@ export interface CodeGenerator {
   generate(node: SemanticNode): string;
 }
 
+/**
+ * Per-language vocabulary for a command added via {@link DomainExtension}.
+ *
+ * This is the same shape a language profile uses, narrowed to the parts a
+ * single new command needs.
+ */
+export interface ExtensionVocabulary {
+  /** The command's verb in this language. */
+  readonly keyword: PatternGenLanguageProfile['keywords'][string];
+  /**
+   * Markers for roles this language does not already cover in its profile.
+   * Usually unnecessary: role markers are shared across a domain's commands.
+   */
+  readonly roleMarkers?: PatternGenLanguageProfile['roleMarkers'];
+}
+
+/**
+ * A command added to a DSL from outside the package that defines it.
+ *
+ * Supplying a schema plus one vocabulary entry per language is enough to parse,
+ * render and translate the command in every one of them — word order, role
+ * markers and keyword placement all follow from the schema and the language
+ * profiles. `render` and `generate` are only needed when the command's surface
+ * form or output cannot be derived.
+ *
+ * @example
+ * ```typescript
+ * const research: DomainExtension = {
+ *   schema: defineCommand({
+ *     action: 'research',
+ *     roles: [
+ *       defineRole({ role: 'patient', required: true, expectedTypes: ['expression'] }),
+ *       defineRole({
+ *         role: 'source', required: true, expectedTypes: ['expression'],
+ *         markerOverride: { en: 'from', ja: 'から', ar: 'من' },
+ *       }),
+ *     ],
+ *   }),
+ *   vocabulary: {
+ *     en: { keyword: { primary: 'research' } },
+ *     ja: { keyword: { primary: '調査' } },
+ *     ar: { keyword: { primary: 'ابحث' } },
+ *   },
+ * };
+ *
+ * const llm = createLLMDSL({ extensions: [research] });
+ * llm.parse('research "climate" from #wiki', 'en');
+ * llm.render(node, 'ja');  // → '"climate" #wiki から 調査'
+ * ```
+ */
+export interface DomainExtension {
+  /** Schema for the new command. Its action must not collide with an existing one. */
+  readonly schema: CommandSchema;
+
+  /**
+   * Vocabulary per language code. Codes must be configured on the DSL; an
+   * unknown code is a configuration error, not a silent no-op.
+   *
+   * A subset is allowed — a command can be added in six of a DSL's eleven
+   * languages. The languages left out simply do not parse it, so vocabulary
+   * can be filled in over time rather than all at once.
+   */
+  readonly vocabulary: Readonly<Record<string, ExtensionVocabulary>>;
+
+  /** Custom rendering for this action; takes precedence over the schema-driven path. */
+  readonly render?: DomainRenderFn;
+
+  /** Custom code generation for this action; takes precedence over the DSL's generator. */
+  readonly generate?: (node: SemanticNode) => string;
+}
+
 /**
  * DSL configuration with dependency injection support.
  */
@@ -69,6 +146,19 @@ export interface DSLConfig {
   /** Code generator for compilation (default: none) */
   readonly codeGenerator?: CodeGenerator;
 
+  /**
+   * The domain's natural-language renderer, consulted by {@link MultilingualDSL.render}.
+   * Actions it returns `null` for fall through to the schema-driven renderer.
+   */
+  readonly renderer?: DomainRenderFn;
+
+  /**
+   * Commands added from outside this package. Their schemas and vocabulary are
+   * merged into the DSL before pattern generation, so they parse, render and
+   * compile like built-in commands.
+   */
+  readonly extensions?: readonly DomainExtension[];
+
   // === Options ===
 
   /** Auto-generate patterns from schemas (default: true) */
@@ -118,6 +208,16 @@ export interface MultilingualDSL {
   // Translation
   translate(input: string, fromLanguage: string, toLanguage: string): string;
 
+  /**
+   * Render a parsed node back to natural language.
+   *
+   * Returns `null` when the action cannot be rendered — no custom renderer, no
+   * domain renderer, and no schema. Optional so that third-party implementers
+   * of this interface are not broken by its addition; `createMultilingualDSL`
+   * always provides it.
+   */
+  render?(node: SemanticNode, language: string): string | null;
+
   // Language support
   getSupportedLanguages(): string[];
 }
@@ -129,9 +229,17 @@ class DSLRegistry {
   private patterns = new Map<string, LanguagePattern[]>();
   private tokenizers = new Map<string, LanguageTokenizer>();
   private schemas: readonly CommandSchema[];
-
-  constructor(config: DSLConfig) {
+  private partiallyTranslatedActions: ReadonlySet<string>;
+
+  /**
+   * @param partiallyTranslatedActions Actions permitted to lack a keyword in
+   *   some configured languages — extension commands, which may be added for a
+   *   subset of the DSL's languages. A built-in command missing a keyword stays
+   *   an error, since that is a domain-authoring mistake.
+   */
+  constructor(config: DSLConfig, partiallyTranslatedActions: ReadonlySet<string> = new Set()) {
     this.schemas = config.schemas;
+    this.partiallyTranslatedActions = partiallyTranslatedActions;
 
     // Register each language
     for (const lang of config.languages) {
@@ -146,6 +254,12 @@ class DSLRegistry {
     // Generate patterns for this language
     const patterns: LanguagePattern[] = [];
     for (const schema of this.schemas) {
+      if (
+        this.partiallyTranslatedActions.has(schema.action) &&
+        !lang.patternProfile.keywords[schema.action]
+      ) {
+        continue; // this extension command was not translated into this language
+      }
       const pattern = generatePattern(schema, lang.patternProfile);
       patterns.push(pattern);
     }
@@ -173,16 +287,42 @@ class MultilingualDSLImpl implements MultilingualDSL {
   private registry: DSLRegistry;
   private matcher: PatternMatcher;
   private transformer: GrammarTransformer;
+  private profileProvider: ProfileProvider;
   private codeGenerator?: CodeGenerator;
   private schemaLookup: SchemaLookup;
-
-  constructor(config: DSLConfig, registry: DSLRegistry, transformer: GrammarTransformer) {
+  private domainRenderer?: DomainRenderFn;
+  private extensionRenderers: Map<string, DomainRenderFn>;
+  private schemaFallbackRenderer: DomainRenderFn;
+
+  constructor(
+    config: DSLConfig,
+    registry: DSLRegistry,
+    transformer: GrammarTransformer,
+    profileProvider: ProfileProvider
+  ) {
     this.registry = registry;
     this.matcher = new PatternMatcher();
     this.transformer = transformer;
+    this.profileProvider = profileProvider;
     if (config.codeGenerator) {
       this.codeGenerator = config.codeGenerator;
     }
+    if (config.renderer) {
+      this.domainRenderer = config.renderer;
+    }
+
+    this.extensionRenderers = new Map();
+    for (const extension of config.extensions ?? []) {
+      if (extension.render) this.extensionRenderers.set(extension.schema.action, extension.render);
+    }
+
+    // Last resort for render(): covers extension commands with no custom
+    // renderer, and any built-in action the domain renderer declines. Lazy, so
+    // a DSL that never renders pays nothing.
+    this.schemaFallbackRenderer = createDomainRenderer({
+      schemas: config.schemas,
+      profiles: config.languages.map(l => l.patternProfile),
+    });
 
     // Build SchemaLookup from config schemas for explicit syntax validation
     const schemaMap = new Map(config.schemas.map(s => [s.action, s]));
@@ -266,6 +406,22 @@ class MultilingualDSLImpl implements MultilingualDSL {
   translate(input: string, fromLanguage: string, toLanguage: string): string {
     // Explicit syntax is language-agnostic — return unchanged
     if (isExplicitSyntax(input)) return input;
+
+    // translate() is the only path that needs grammar profiles; parse/validate/
+    // compile work without them. `grammarProfile` is optional on LanguageConfig,
+    // so omitting it fails here rather than at construction — name the missing
+    // field instead of leaving the transformer's bare "No profile found".
+    for (const language of [fromLanguage, toLanguage]) {
+      if (!this.profileProvider.getProfile(language)) {
+        throw new Error(
+          `translate() requires a grammar profile for language "${language}", but none is ` +
+            `configured. Set 'grammarProfile' on the LanguageConfig for "${language}" in ` +
+            `createMultilingualDSL() (parse/validate/compile do not need it), or inject a ` +
+            `custom 'profileProvider'.`
+        );
+      }
+    }
+
     // Use injected grammar transformer
     return this.transformer.transform(input, fromLanguage, toLanguage);
   }
@@ -299,6 +455,24 @@ class MultilingualDSLImpl implements MultilingualDSL {
     }
   }
 
+  render(node: SemanticNode, language: string): string | null {
+    // 1. An extension's own renderer, if it supplied one
+    const extensionRenderer = this.extensionRenderers.get(node.action);
+    if (extensionRenderer) {
+      const rendered = extensionRenderer(node, language);
+      if (rendered != null) return rendered;
+    }
+
+    // 2. The domain's renderer — its hand-written cases are authoritative
+    if (this.domainRenderer) {
+      const rendered = this.domainRenderer(node, language);
+      if (rendered != null) return rendered;
+    }
+
+    // 3. Schema-driven, which is how extension commands render by default
+    return this.schemaFallbackRenderer(node, language);
+  }
+
   getSupportedLanguages(): string[] {
     return this.registry.getSupportedLanguages();
   }
@@ -369,12 +543,107 @@ function createDefaultProfileProvider(config: DSLConfig): ProfileProvider {
   return new InMemoryProfileProvider(profiles);
 }
 
+/**
+ * Fold extensions into the config so that everything downstream — pattern
+ * generation, the dictionary, explicit-syntax lookup, rendering — sees
+ * extension commands exactly as it sees built-in ones.
+ *
+ * Done up front rather than by mutating a live DSL: `DSLRegistry` generates
+ * every language's patterns in its constructor, so a command registered later
+ * would compile and render but silently fail to parse.
+ */
+function applyExtensions(config: DSLConfig): DSLConfig {
+  const extensions = config.extensions;
+  if (!extensions || extensions.length === 0) return config;
+
+  const configuredLanguages = new Set(config.languages.map(l => l.code));
+  const actions = new Set(config.schemas.map(s => s.action));
+
+  for (const extension of extensions) {
+    const { action } = extension.schema;
+    if (actions.has(action)) {
+      throw new Error(
+        `Extension command "${action}" collides with a command this DSL already defines. ` +
+          `Pick a different action name.`
+      );
+    }
+    actions.add(action);
+
+    for (const code of Object.keys(extension.vocabulary)) {
+      if (!configuredLanguages.has(code)) {
+        throw new Error(
+          `Extension command "${action}" supplies vocabulary for language "${code}", which is ` +
+            `not configured on this DSL. Configured languages: ` +
+            `${[...configuredLanguages].join(', ')}.`
+        );
+      }
+    }
+  }
+
+  const schemas = [...config.schemas, ...extensions.map(e => e.schema)];
+
+  const languages = config.languages.map(lang => {
+    const keywords = { ...lang.patternProfile.keywords };
+    let roleMarkers = lang.patternProfile.roleMarkers;
+
+    for (const extension of extensions) {
+      const vocabulary = extension.vocabulary[lang.code];
+      if (!vocabulary) continue;
+      keywords[extension.schema.action] = { ...vocabulary.keyword };
+      if (vocabulary.roleMarkers) {
+        roleMarkers = { ...roleMarkers, ...vocabulary.roleMarkers };
+      }
+    }
+
+    return {
+      ...lang,
+      patternProfile: {
+        ...lang.patternProfile,
+        keywords,
+        ...(roleMarkers !== undefined && { roleMarkers }),
+      },
+    };
+  });
+
+  // Extension code generators dispatch ahead of the domain's own generator,
+  // whose `default` branch would otherwise throw on an action it never knew.
+  const extensionGenerators = new Map(
+    extensions.filter(e => e.generate).map(e => [e.schema.action, e.generate!])
+  );
+  const baseGenerator = config.codeGenerator;
+  const codeGenerator: CodeGenerator | undefined =
+    extensionGenerators.size > 0
+      ? {
+          generate(node: SemanticNode): string {
+            const generate = extensionGenerators.get(node.action);
+            if (generate) return generate(node);
+            if (!baseGenerator) {
+              throw new Error(`No code generator for action "${node.action}"`);
+            }
+            return baseGenerator.generate(node);
+          },
+        }
+      : baseGenerator;
+
+  return {
+    ...config,
+    schemas,
+    languages,
+    ...(codeGenerator !== undefined && { codeGenerator }),
+  };
+}
+
 export function createMultilingualDSL(config: DSLConfig): MultilingualDSL {
+  // Extensions must be folded in before anything reads the config
+  const effectiveConfig = applyExtensions(config);
+  const extensionActions = new Set((config.extensions ?? []).map(e => e.schema.action));
+
   // Create or use provided dictionary
-  const dictionary = config.dictionary ?? createDefaultDictionary(config);
+  const dictionary = effectiveConfig.dictionary ?? createDefaultDictionary(effectiveConfig);
 
   // Create or use provided profile provider
-  const profileProvider = config.profileProvider ?? createDefaultProfileProvider(config);
+  const profileProvider =
+    effectiveConfig.profileProvider ?? createDefaultProfileProvider(effectiveConfig);
 
   // Create grammar transformer with injected dependencies
   const transformer = new GrammarTransformer({
@@ -383,6 +652,6 @@ export function createMultilingualDSL(config: DSLConfig): MultilingualDSL {
   });
 
   // Create registry and implementation
-  const registry = new DSLRegistry(config);
-  return new MultilingualDSLImpl(config, registry, transformer);
+  const registry = new DSLRegistry(effectiveConfig, extensionActions);
+  return new MultilingualDSLImpl(effectiveConfig, registry, transformer, profileProvider);
 }

package/src/api/domain-registry.ts

@@ -30,9 +30,8 @@
  * ```
  */
 
-import type { SemanticNode } from '../core/types';
 import type { MultilingualDSL, CompileResult, ValidationResult } from './create-dsl';
-import type { NaturalLanguageRenderer } from '../generation/renderer';
+import type { NaturalLanguageRenderer, DomainRenderFn } from '../generation/renderer';
 import type { CommandSchema } from '../schema/command-schema';
 import type { SchemaLookup } from '../ir/types';
 import type { Diagnostic } from '../generation/diagnostics';
@@ -82,11 +81,12 @@ export interface DomainDescriptor {
   /**
    * Factory to lazily create the renderer.
    * Required for translate_{name} tool. Called once on first use.
+   *
+   * The function form may return `null` for an action it cannot render; the
+   * translate tool reports that as an unsuccessful translation.
    */
   readonly getRenderer?: () =>
-    | NaturalLanguageRenderer
-    | ((node: SemanticNode, language: string) => string)
-    | Promise<NaturalLanguageRenderer | ((node: SemanticNode, language: string) => string)>;
+    NaturalLanguageRenderer | DomainRenderFn | Promise<NaturalLanguageRenderer | DomainRenderFn>;
 
   /**
    * Which standard tools to generate. Default: all four.
@@ -154,10 +154,7 @@ export interface MCPToolResponse {
 export class DomainRegistry {
   private descriptors = new Map<string, DomainDescriptor>();
   private dslCache = new Map<string, MultilingualDSL>();
-  private rendererCache = new Map<
-    string,
-    NaturalLanguageRenderer | ((node: SemanticNode, language: string) => string)
-  >();
+  private rendererCache = new Map<string, NaturalLanguageRenderer | DomainRenderFn>();
 
   /**
    * Register a domain.
@@ -361,7 +358,7 @@ export class DomainRegistry {
 
   private async getRenderer(
     descriptor: DomainDescriptor
-  ): Promise<NaturalLanguageRenderer | ((node: SemanticNode, language: string) => string) | null> {
+  ): Promise<NaturalLanguageRenderer | DomainRenderFn | null> {
     if (!descriptor.getRenderer) return null;
 
     const cached = this.rendererCache.get(descriptor.name);
@@ -466,6 +463,14 @@ export class DomainRegistry {
       } catch {
         // Fall through with rendered = null
       }
+    } else if (dsl.render) {
+      // A descriptor without an explicit renderer can still render: the DSL
+      // knows its own domain renderer and its schema fallback.
+      try {
+        rendered = dsl.render(node, to);
+      } catch {
+        // Fall through with rendered = null
+      }
     }
 
     const roles: Record<string, unknown> = {};