@lokascript/framework 2.7.1 → 2.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lokascript/framework",
-  "version": "2.7.1",
+  "version": "2.8.0",
   "description": "Generic framework for building multilingual DSLs with semantic parsing and grammar transformation",
   "type": "module",
   "main": "dist/index.cjs",
@@ -96,7 +96,7 @@
   "author": "LokaScript Contributors",
   "license": "MIT",
   "dependencies": {
-    "@lokascript/intent": "^2.7.1"
+    "@lokascript/intent": "^2.8.0"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",

package/src/core/tokenization/base-tokenizer.ts

@@ -347,7 +347,61 @@ export abstract class BaseTokenizer implements LanguageTokenizer {
       }
     }
 
-    return new TokenStreamImpl(tokens, this.language);
+    return new TokenStreamImpl(this.mergeColonQualifiedNames(tokens), this.language);
+  }
+
+  /**
+   * ASCII word of the shape the English word-walker produces. Excludes `:`, so a
+   * token that already carries a qualifier never merges again — `a:b:c` yields
+   * `a:b` + `:c`, byte-matching the English extractor's single-segment merge.
+   */
+  private static readonly ASCII_WORD = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+  /** `:name` — only a variable-ref-style extractor ever emits this token shape. */
+  private static readonly COLON_QUALIFIER = /^:[A-Za-z_][A-Za-z0-9_]*$/;
+
+  /**
+   * Fuse `name` + `:qualifier` into ONE identifier (`draggable:start`).
+   *
+   * `:name` is hyperscript's local-variable sigil, but a colon IMMEDIATELY
+   * preceded by an identifier is a qualifier (custom event namespace), not a
+   * sigil. The English tokenizer already merges these inside
+   * EnglishKeywordExtractor; this post-pass gives the other 23 languages the
+   * same stream. Strict position adjacency is the discriminator: whitespace
+   * between the tokens (`trigger :start`) breaks `end === start`, so a spaced
+   * local-variable reference survives untouched.
+   *
+   * Self-gating for non-hyperscript tokenizers (domain DSLs): their extractor
+   * sets tokenize `:` as bare punctuation (length 1), which never matches
+   * COLON_QUALIFIER, so this pass is a no-op for them.
+   */
+  protected mergeColonQualifiedNames(tokens: LanguageToken[]): LanguageToken[] {
+    const out: LanguageToken[] = [];
+    for (const tok of tokens) {
+      const prev = out[out.length - 1];
+      // No kind gate: the English extractor merges before classification, so a
+      // word some language classifies as particle/keyword (es `a`, tr `i`)
+      // must fuse the same way. ASCII_WORD already excludes every non-word
+      // kind structurally (selectors, urls, numbers, strings, operators).
+      if (
+        prev &&
+        BaseTokenizer.ASCII_WORD.test(prev.value) &&
+        BaseTokenizer.COLON_QUALIFIER.test(tok.value) &&
+        prev.position.end === tok.position.start
+      ) {
+        const merged = prev.value + tok.value;
+        // Re-classify and drop normalized/stem/metadata — the merged word is no
+        // longer the keyword the pieces may have been (matches the en shape).
+        out[out.length - 1] = createToken(
+          merged,
+          this.classifyToken(merged),
+          createPosition(prev.position.start, tok.position.end)
+        );
+        continue;
+      }
+      out.push(tok);
+    }
+    return out;
   }
 
   /**

package/src/core/tokenization/colon-qualifier.test.ts

@@ -0,0 +1,129 @@
+/**
+ * Contract tests for BaseTokenizer.mergeColonQualifiedNames().
+ *
+ * `:name` is hyperscript's local-variable sigil, but a colon immediately
+ * preceded by an identifier is a qualifier (custom event namespace:
+ * `draggable:start`). The English tokenizer merges these inside its keyword
+ * extractor; the post-pass gives every other language the same stream. The
+ * merge must fire only on strict adjacency and only on `:name`-shaped tokens —
+ * domain DSL tokenizers (SQL, BDD, …) emit `:` as bare punctuation, which the
+ * pass must never touch.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { BaseTokenizer, createSimpleTokenizer } from './base-tokenizer';
+import type { TokenKind } from '../types';
+import type { ValueExtractor, ExtractionResult } from '../../interfaces/value-extractor';
+
+/** Minimal `:name`/`$name`/`^name` extractor (shape of semantic's VariableRefExtractor). */
+class SigilRefExtractor implements ValueExtractor {
+  readonly name = 'sigil-ref';
+
+  canExtract(input: string, position: number): boolean {
+    const ch = input[position];
+    return (
+      (ch === ':' || ch === '$' || ch === '^') &&
+      position + 1 < input.length &&
+      /[a-zA-Z_]/.test(input[position + 1])
+    );
+  }
+
+  extract(input: string, position: number): ExtractionResult | null {
+    if (!this.canExtract(input, position)) return null;
+    let length = 1;
+    while (position + length < input.length && /[a-zA-Z0-9_]/.test(input[position + length])) {
+      length++;
+    }
+    return { value: input.substring(position, position + length), length };
+  }
+}
+
+/** Minimal ASCII word extractor (shape of the per-language word walkers). */
+class WordExtractor implements ValueExtractor {
+  readonly name = 'word';
+
+  canExtract(input: string, position: number): boolean {
+    return /[a-zA-Z_]/.test(input[position]);
+  }
+
+  extract(input: string, position: number): ExtractionResult | null {
+    let length = 0;
+    while (position + length < input.length && /[a-zA-Z0-9_]/.test(input[position + length])) {
+      length++;
+    }
+    return length > 0 ? { value: input.substring(position, position + length), length } : null;
+  }
+}
+
+const KEYWORDS = new Set(['trigger', 'click']);
+
+class ProbeTokenizer extends BaseTokenizer {
+  readonly language = 'xx';
+  readonly direction = 'ltr' as const;
+
+  constructor() {
+    super();
+    this.registerExtractors([new SigilRefExtractor(), new WordExtractor()]);
+  }
+
+  classifyToken(token: string): TokenKind {
+    return KEYWORDS.has(token) ? 'keyword' : 'identifier';
+  }
+}
+
+function values(
+  tokenizer: { tokenize(input: string): { tokens: readonly { value: string }[] } },
+  input: string
+): string[] {
+  return tokenizer.tokenize(input).tokens.map(t => t.value);
+}
+
+describe('mergeColonQualifiedNames', () => {
+  const t = new ProbeTokenizer();
+
+  it('fuses identifier + :qualifier into one token', () => {
+    expect(values(t, 'trigger draggable:start')).toEqual(['trigger', 'draggable:start']);
+  });
+
+  it('re-classifies the merged token and spans both positions', () => {
+    const tokens = t.tokenize('trigger draggable:start').tokens;
+    const merged = tokens[1];
+    expect(merged.kind).toBe('identifier');
+    expect(merged.position.start).toBe('trigger '.length);
+    expect(merged.position.end).toBe('trigger draggable:start'.length);
+  });
+
+  it('fuses when the pre-colon word is a keyword (en parity: merge before lookup)', () => {
+    const tokens = t.tokenize('click:foo').tokens;
+    expect(tokens.map(x => x.value)).toEqual(['click:foo']);
+    expect(tokens[0].kind).toBe('identifier');
+  });
+
+  it('does not fuse across whitespace — spaced :name stays a local-variable ref', () => {
+    expect(values(t, 'trigger :start')).toEqual(['trigger', ':start']);
+  });
+
+  it('leaves a leading bare sigil untouched', () => {
+    expect(values(t, ':start')).toEqual([':start']);
+  });
+
+  it('merges a single segment only — a:b:c matches the English extractor', () => {
+    expect(values(t, 'a:b:c')).toEqual(['a:b', ':c']);
+  });
+
+  it('never touches $ and ^ sigils', () => {
+    expect(values(t, 'put $foo into ^bar')).toEqual(['put', '$foo', 'into', '^bar']);
+  });
+
+  it('is a no-op for domain-DSL streams where : is bare punctuation', () => {
+    const sql = createSimpleTokenizer({
+      language: 'en',
+      keywords: ['select', 'from', 'where'],
+      includeOperators: true,
+    });
+    // Named params never fuse: the colon tokenizes as length-1 punctuation,
+    // which can never match the :name qualifier shape.
+    expect(values(sql, 'WHERE x = :param')).toEqual(['WHERE', 'x', '=', ':', 'param']);
+    expect(values(sql, 'x=:param')).toEqual(['x', '=', ':', 'param']);
+  });
+});

package/src/core/tokenization/extractors.ts

@@ -34,6 +34,12 @@ import {
  * Method call handling:
  * - #dialog.showModal() → stops after #dialog (method call, not compound selector)
  * - #box.active → compound selector (no parens)
+ *
+ * NOTE: intentionally diverges from the semantic package's copy
+ * (packages/semantic/src/tokenizers/extractors/css-selector.ts), which also
+ * consumes pseudo-class/pseudo-element segments (#x:hover, .a:not(.b)). This
+ * legacy version is only used by BaseTokenizer.trySelector (no semantic call
+ * sites) and stays as-is.
  */
 export function extractCssSelector(input: string, startPos: number): string | null {
   if (startPos >= input.length) return null;

package/src/generation/pattern-generator.ts

@@ -272,7 +272,11 @@ function buildFormatString(
     parts.push(keyword);
   }
 
-  for (const role of schema.roles) {
+  // Order roles the same way buildTokens does (descending position). Iterating
+  // declaration order here would disagree with the token order whenever a
+  // schema's roles are declared in a different order than their positions imply.
+  const sortedRoles = sortRolesByWordOrder(schema.roles, profile.wordOrder);
+  for (const role of sortedRoles) {
     const marker = getMarkerForRole(role, profile);
     const roleName = `{${role.role}}`;
 

package/src/ir/protocol-json.test.ts

@@ -1036,6 +1036,27 @@ describe('v1.2: diagnostics', () => {
     expect(node.diagnostics).toHaveLength(1);
     expect(node.diagnostics![0].code).toBe('SCHEMA_VALUE_TYPE_MISMATCH');
     expect(node.diagnostics![0].severity).toBe('error');
+    expect(node.diagnostics![0].role).toBe('patient');
+  });
+
+  // `role` is required by the wire format's Diagnostic shape. Both directions
+  // used to drop it, and the fixtures happened to omit it, so nothing noticed.
+  it('preserves a diagnostic `role` across a protocol-JSON round-trip', () => {
+    const json: ProtocolNodeJSON = {
+      kind: 'command',
+      action: 'toggle',
+      roles: { patient: { type: 'literal', value: 'hello', dataType: 'string' } },
+      diagnostics: [
+        {
+          level: 'error',
+          role: 'patient',
+          message: "toggle.patient expects type [selector], got 'literal'",
+          code: 'SCHEMA_VALUE_TYPE_MISMATCH',
+        },
+      ],
+    };
+    expect(fromProtocolJSON(json).diagnostics![0].role).toBe('patient');
+    expect(toProtocolJSON(fromProtocolJSON(json))).toEqual(json);
   });
 
   describe('fixture conformance: type-constraints.json', () => {

package/src/ir/references.test.ts

@@ -10,6 +10,9 @@ describe('isValidReference', () => {
     expect(isValidReference('event')).toBe(true);
     expect(isValidReference('target')).toBe(true);
     expect(isValidReference('body')).toBe(true);
+    expect(isValidReference('document')).toBe(true);
+    expect(isValidReference('window')).toBe(true);
+    expect(isValidReference('detail')).toBe(true);
   });
 
   it('rejects non-references', () => {
@@ -32,7 +35,7 @@ describe('isValidReference', () => {
     expect(isValidReference('it', custom)).toBe(false);
   });
 
-  it('DEFAULT_REFERENCES contains exactly 7 entries', () => {
-    expect(DEFAULT_REFERENCES.size).toBe(7);
+  it('DEFAULT_REFERENCES contains exactly 10 entries', () => {
+    expect(DEFAULT_REFERENCES.size).toBe(10);
   });
 });

package/src/prompts/prompt-generator.ts

@@ -165,7 +165,9 @@ function buildProtocolSection(): PromptSection {
 3. No spaces around the colon in role:value pairs
 4. Strings with spaces must be quoted: \`patient:"hello world"\`
 5. Selectors start with \`#\`, \`.\`, \`[\`, \`@\`, or \`*\`
-6. Output must be valid bracket syntax: \`[action role:value ...]\``;
+6. A selector containing a space, a combinator (\`>\` \`+\` \`~\`), or a comma must use a selector literal: \`patient:<ul > li/>\`, \`patient:<.a, .b/>\`
+7. Inside a structural role (\`body\`, \`then\`, \`else\`, \`condition\`, \`loop-body\`, \`variable\`, \`catch\`, \`finally\`) a \`[...]\` value is always a nested command. Write an attribute selector there as \`condition:<[data-active]/>\`
+8. Output must be valid bracket syntax: \`[action role:value ...]\``;
 
   return {
     id: 'protocol',
@@ -180,6 +182,7 @@ function buildValueTypeSection(): PromptSection {
 
 | Type | Syntax | Example |
 |------|--------|---------|
+| Selector literal | Delimited by \`<\` and \`/>\` | \`<ul > li/>\`, \`<.a, .b/>\`, \`<[data-id]/>\` |
 | Selector | Starts with \`#\` \`.\` \`[\` \`@\` \`*\` | \`#button\`, \`.active\`, \`[data-id]\` |
 | String | Quoted with \`"\` or \`'\` | \`"hello world"\`, \`'json'\` |
 | Boolean | Exact: \`true\` / \`false\` | \`visible:true\` |