@lokascript/framework 4.5.0 → 4.7.0

package/src/core/matching/stock-confidence.ts

@@ -0,0 +1,238 @@
+/**
+ * The stock confidence model: how framework's old matcher scored a domain
+ * pattern's match (`core/pattern-matching/`), moved here as a `ConfidenceModel`
+ * for the matcher's skeleton (domains rethink Phase 4.7); the old matcher calls
+ * it too until Phase 6.
+ *
+ * Role coverage (a required slot 1, an optional one 0.8, an absent optional one
+ * with a default 60% of that), a stem-match penalty (up to 15%, floor 0.5), and
+ * two Arabic adjustments: a VSO verb-first boost and the preposition each role
+ * takes most naturally.
+ */
+
+import type { LanguagePattern, SemanticRole, SemanticValue } from '../types';
+import type { ConfidenceBreakdown, ConfidenceContext, ConfidenceModel } from './confidence-model';
+
+/**
+ * Calculate confidence score for a match (0-1).
+ *
+ * Confidence is reduced for:
+ * - Stem matches (morphological normalization has inherent uncertainty)
+ * - Missing optional roles (but less penalty if role has a default value)
+ *
+ * Confidence is increased for:
+ * - VSO languages (Arabic) when pattern starts with a verb
+ */
+function scoreParts(ctx: ConfidenceContext): ConfidenceBreakdown {
+  const { pattern, captured } = ctx;
+  let score = 0;
+  let maxScore = 0;
+
+  // Helper to check if a role has a default value in extraction rules
+  const hasDefault = (role: SemanticRole): boolean => {
+    return pattern.extraction?.[role]?.default !== undefined;
+  };
+
+  // Score based on captured roles
+  for (const token of pattern.template.tokens) {
+    if (token.type === 'role') {
+      maxScore += 1;
+      if (captured.has(token.role)) {
+        score += 1;
+      }
+    } else if (token.type === 'group') {
+      // Group tokens are optional - weight depends on whether they have defaults
+      for (const subToken of token.tokens) {
+        if (subToken.type === 'role') {
+          const roleHasDefault = hasDefault(subToken.role);
+          const weight = 0.8; // Optional roles: 80% weight
+          maxScore += weight;
+
+          if (captured.has(subToken.role)) {
+            // Role was explicitly provided by user
+            score += weight;
+          } else if (roleHasDefault) {
+            // Role has a default - give 60% partial credit since command is semantically complete
+            // This prevents penalizing common patterns like "toggle .active" (default: me)
+            score += weight * 0.6;
+          }
+          // If no default and not captured, score += 0 (true penalty for missing info)
+        }
+      }
+    }
+  }
+
+  const roleCoverage = maxScore > 0 ? score / maxScore : 1;
+  let baseConfidence = roleCoverage;
+  let stemPenalty = 0;
+
+  // Apply penalty for stem matches
+  // Each stem match reduces confidence slightly (e.g., 5% per stem match)
+  // This ensures exact matches are preferred over morphological matches
+  if (ctx.stemMatchCount > 0 && ctx.totalKeywordMatches > 0) {
+    stemPenalty = (ctx.stemMatchCount / ctx.totalKeywordMatches) * 0.15;
+    baseConfidence = Math.max(0.5, baseConfidence - stemPenalty);
+  }
+
+  // Apply VSO confidence boost for Arabic verb-first patterns
+  const vsoBoost = calculateVSOConfidenceBoost(pattern);
+  baseConfidence = Math.min(1.0, baseConfidence + vsoBoost);
+
+  // Apply preposition disambiguation adjustment for Arabic
+  const prepositionAdjustment = arabicPrepositionDisambiguation(pattern, captured);
+  baseConfidence = Math.max(0.0, Math.min(1.0, baseConfidence + prepositionAdjustment));
+
+  return {
+    roleCoverage,
+    stemPenalty,
+    languageBoost: vsoBoost,
+    languageAdjustment: prepositionAdjustment,
+    final: baseConfidence,
+  };
+}
+
+/**
+ * Calculate confidence boost for VSO (Verb-Subject-Object) language patterns.
+ * Arabic naturally uses VSO word order, so patterns that start with a verb
+ * should receive a confidence boost.
+ *
+ * Returns +0.15 confidence boost if:
+ * - Language is Arabic ('ar')
+ * - Pattern's first token is a verb keyword
+ *
+ * @param pattern The language pattern being matched
+ * @returns Confidence boost (0 or 0.15)
+ */
+function calculateVSOConfidenceBoost(pattern: LanguagePattern): number {
+  // Only apply to Arabic
+  if (pattern.language !== 'ar') {
+    return 0;
+  }
+
+  // Check if first token in pattern is a literal (keyword)
+  const firstToken = pattern.template.tokens[0];
+  if (!firstToken || firstToken.type !== 'literal') {
+    return 0;
+  }
+
+  // List of Arabic verb keywords (command verbs)
+  const ARABIC_VERBS = new Set([
+    'بدل',
+    'غير',
+    'أضف',
+    'أزل',
+    'ضع',
+    'اجعل',
+    'عين',
+    'زد',
+    'انقص',
+    'سجل',
+    'أظهر',
+    'أخف',
+    'شغل',
+    'أرسل',
+    'ركز',
+    'شوش',
+    'توقف',
+    'انسخ',
+    'احذف',
+    'اصنع',
+    'انتظر',
+    'انتقال',
+    'أو',
+  ]);
+
+  // Check if first token value is a verb
+  if (ARABIC_VERBS.has(firstToken.value)) {
+    return 0.15;
+  }
+
+  // Check alternatives
+  if (firstToken.alternatives) {
+    for (const alt of firstToken.alternatives) {
+      if (ARABIC_VERBS.has(alt)) {
+        return 0.15;
+      }
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Arabic preposition disambiguation for confidence adjustment.
+ *
+ * Different Arabic prepositions are more or less natural for different semantic roles:
+ * - على (on/upon) is preferred for patient/target roles (element selectors)
+ * - إلى (to) is preferred for destination roles
+ * - من (from) is preferred for source roles
+ * - في (in) is preferred for location roles
+ *
+ * This method analyzes the prepositions used with captured semantic roles and
+ * adjusts confidence based on idiomaticity:
+ * - +0.10 for highly idiomatic preposition choices
+ * - -0.10 for less natural preposition choices
+ *
+ * @param pattern The language pattern being matched
+ * @param captured The captured semantic values
+ * @returns Confidence adjustment (-0.10 to +0.10)
+ */
+function arabicPrepositionDisambiguation(
+  pattern: LanguagePattern,
+  captured: ReadonlyMap<SemanticRole, SemanticValue>
+): number {
+  // Only apply to Arabic
+  if (pattern.language !== 'ar') {
+    return 0;
+  }
+
+  let adjustment = 0;
+
+  // Preferred prepositions for each semantic role
+  // Only including roles that commonly use prepositions in Arabic
+  const PREFERRED_PREPOSITIONS: Partial<Record<SemanticRole, string[]>> = {
+    patient: ['على'], // element selectors prefer على (on/upon)
+    destination: ['إلى', 'الى'], // destination prefers إلى (to)
+    source: ['من'], // source prefers من (from)
+    agent: ['من'], // agent/by prefers من (from/by)
+    manner: ['ب'], // manner prefers ب (with/by)
+    style: ['ب'], // style prefers ب (with)
+    goal: ['إلى', 'الى'], // target state prefers إلى (to)
+    method: ['ب'], // method prefers ب (with/by)
+  };
+
+  // Check each captured role for preposition metadata
+  for (const [role, value] of captured.entries()) {
+    // Skip if no preferred prepositions defined for this role
+    const preferred = PREFERRED_PREPOSITIONS[role];
+    if (!preferred || preferred.length === 0) {
+      continue;
+    }
+
+    // Check if the value has preposition metadata (from Arabic tokenizer)
+    // This metadata is attached when a preposition particle token is consumed
+    const metadata =
+      'metadata' in value ? (value as { metadata: Record<string, unknown> }).metadata : undefined;
+    if (metadata && typeof metadata.prepositionValue === 'string') {
+      const usedPreposition = metadata.prepositionValue;
+
+      // Check if the used preposition is in the preferred list
+      if (preferred.includes(usedPreposition)) {
+        // Idiomatic choice - boost confidence
+        adjustment += 0.1;
+      } else {
+        // Less natural choice - reduce confidence
+        adjustment -= 0.1;
+      }
+    }
+  }
+
+  // Cap total adjustment at ±0.10 (even if multiple roles analyzed)
+  return Math.max(-0.1, Math.min(0.1, adjustment));
+}
+
+/** The stock confidence model (stateless: one instance serves every matcher). */
+export const stockConfidenceModel: ConfidenceModel = {
+  calculate: ctx => scoreParts(ctx).final,
+  calculateWithBreakdown: ctx => scoreParts(ctx),
+};

package/src/core/matching/stock-reader.test.ts

@@ -0,0 +1,181 @@
+/**
+ * The stock value reader on the matcher's skeleton (domains rethink Phase 4.7):
+ * each kind of value it reads, a greedy slot's stop markers, the guard against a
+ * marker read as a value, and the skip of an optional slot.
+ */
+import { describe, expect, it } from 'vitest';
+import { MatchSkeleton } from './match-skeleton';
+import { createStockReader, greedyStopMarkers, readGreedy } from './stock-reader';
+import { stockConfidenceModel } from './stock-confidence';
+import { TokenStreamImpl } from '../tokenization/token-utils';
+import type { LanguagePattern, LanguageToken, PatternToken, TokenKind } from '../types';
+
+function stream(...tokens: Array<[string, TokenKind] | [string, TokenKind, string]>) {
+  let offset = 0;
+  const out: LanguageToken[] = tokens.map(([value, kind, normalized]) => {
+    const token: LanguageToken = {
+      value,
+      kind,
+      position: { start: offset, end: offset + value.length },
+      ...(normalized ? { normalized } : {}),
+    };
+    offset += value.length + 1;
+    return token;
+  });
+  return new TokenStreamImpl(out, 'en');
+}
+
+const lit = (value: string, ...alternatives: string[]): PatternToken =>
+  alternatives.length ? { type: 'literal', value, alternatives } : { type: 'literal', value };
+const slot = (
+  role: string,
+  extra: { optional?: boolean; greedy?: boolean; expectedTypes?: string[] } = {}
+): PatternToken => ({ type: 'role', role, ...extra }) as PatternToken;
+const group = (...tokens: PatternToken[]): PatternToken => ({
+  type: 'group',
+  optional: true,
+  tokens,
+});
+
+function pattern(tokens: PatternToken[], language = 'en'): LanguagePattern {
+  return {
+    id: 'p',
+    language,
+    command: 'go',
+    priority: 100,
+    template: { format: 'p', tokens },
+    extraction: {},
+  };
+}
+
+const matcher = () => new MatchSkeleton(createStockReader(), stockConfidenceModel);
+const read = (p: LanguagePattern, ...tokens: Parameters<typeof stream>) => {
+  const result = matcher().matchPattern(stream(...tokens), p);
+  return result && Object.fromEntries(result.captured);
+};
+
+describe('one value per slot', () => {
+  const go = pattern([lit('go'), slot('x')]);
+  it.each([
+    ['#box', 'selector', { type: 'selector', value: '#box' }],
+    ['"hello there"', 'literal', { type: 'literal', value: 'hello there', dataType: 'string' }],
+    ['42', 'literal', { type: 'literal', value: 42, dataType: 'number' }],
+    ['true', 'literal', { type: 'literal', value: true, dataType: 'boolean' }],
+    ['300ms', 'literal', { type: 'literal', value: '300ms', dataType: 'duration' }],
+    ['/api/users', 'url', { type: 'literal', value: '/api/users', dataType: 'string' }],
+    ['users', 'identifier', { type: 'expression', raw: 'users' }],
+  ] as const)('reads %s', (value, kind, want) => {
+    expect(read(go, ['go', 'keyword'], [value, kind])).toEqual({ x: want });
+  });
+
+  it('skips an English article before a selector', () => {
+    expect(read(go, ['go', 'keyword'], ['the', 'identifier'], ['#box', 'selector'])).toEqual({
+      x: { type: 'selector', value: '#box' },
+    });
+  });
+
+  it('refuses a value of a type the slot does not take, unless the slot is optional', () => {
+    const selectorOnly = pattern([lit('go'), slot('x', { expectedTypes: ['selector'] })]);
+    expect(read(selectorOnly, ['go', 'keyword'], ['users', 'identifier'])).toBeNull();
+    const optional = pattern([
+      lit('go'),
+      slot('x', { expectedTypes: ['selector'], optional: true }),
+    ]);
+    expect(read(optional, ['go', 'keyword'], ['users', 'identifier'])).toEqual({});
+  });
+});
+
+describe('a greedy slot', () => {
+  it('reads to the next marker the pattern owes', () => {
+    const select = pattern([
+      lit('select'),
+      slot('columns', { greedy: true }),
+      lit('from'),
+      slot('source'),
+    ]);
+    expect(
+      read(
+        select,
+        ['select', 'keyword'],
+        ['name', 'identifier'],
+        [',', 'punctuation'],
+        ['email', 'identifier'],
+        ['from', 'keyword'],
+        ['users', 'identifier']
+      )
+    ).toEqual({
+      columns: { type: 'expression', raw: 'name , email' },
+      source: { type: 'expression', raw: 'users' },
+    });
+  });
+
+  it('reads to the end when nothing is owed, and is refused when it reads nothing', () => {
+    const where = pattern([lit('where'), slot('condition', { greedy: true })]);
+    expect(
+      read(where, ['where', 'keyword'], ['age', 'identifier'], ['>', 'operator'], ['18', 'literal'])
+    ).toEqual({ condition: { type: 'expression', raw: 'age > 18' } });
+    expect(read(where, ['where', 'keyword'])).toBeNull();
+  });
+
+  it('stops past an optional group at the required literal after it, by value or normalized form', () => {
+    const rest: PatternToken[] = [group(lit('where'), slot('c')), lit('選択', 'select')];
+    expect([...greedyStopMarkers(rest)]).toEqual(['where', '選択', 'select']);
+    const tokens = stream(['users', 'identifier'], ['SELECT', 'keyword', 'select']);
+    expect(readGreedy(tokens, greedyStopMarkers(rest))).toEqual({
+      type: 'expression',
+      raw: 'users',
+    });
+    expect(tokens.position()).toBe(1);
+  });
+});
+
+describe('the marker guard and the optional skip', () => {
+  it('a required slot does not read the marker the pattern wants next', () => {
+    // SOV `{patient} を 実行` given `を 実行`: the patient is missing, not `を`.
+    const sov = pattern([slot('patient'), lit('を'), lit('実行')], 'ja');
+    expect(read(sov, ['を', 'particle'], ['実行', 'keyword'])).toBeNull();
+  });
+
+  it('an optional slot is skipped for the literal the pattern wants next, when the rest then reads to the end', () => {
+    const p = pattern([lit('go'), group(slot('x', { optional: true })), lit('now')]);
+    expect(read(p, ['go', 'keyword'], ['now', 'keyword'])).toEqual({});
+    expect(read(p, ['go', 'keyword'], ['#a', 'selector'], ['now', 'keyword'])).toEqual({
+      x: { type: 'selector', value: '#a' },
+    });
+  });
+});
+
+describe('a required slot takes back the optional slot before it', () => {
+  it('when the optional slot took its value (behaviorspec `when clicks`)', () => {
+    const when = pattern([lit('when'), group(slot('actor', { optional: true })), slot('action')]);
+    expect(read(when, ['when', 'keyword'], ['clicks', 'identifier'])).toEqual({
+      action: { type: 'expression', raw: 'clicks' },
+    });
+    expect(
+      read(when, ['when', 'keyword'], ['user', 'identifier'], ['clicks', 'identifier'])
+    ).toEqual({
+      actor: { type: 'expression', raw: 'user' },
+      action: { type: 'expression', raw: 'clicks' },
+    });
+  });
+
+  it('when it stands at its own marker (flow `orders として 取得変数`)', () => {
+    const capture = pattern(
+      [
+        group(slot('patient', { optional: true })),
+        slot('destination'),
+        lit('として'),
+        lit('取得変数'),
+      ],
+      'ja'
+    );
+    expect(
+      read(capture, ['orders', 'identifier'], ['として', 'particle'], ['取得変数', 'keyword'])
+    ).toEqual({ destination: { type: 'expression', raw: 'orders' } });
+  });
+
+  it('not when something was read between them', () => {
+    const p = pattern([group(slot('a', { optional: true })), lit('to'), slot('b')]);
+    expect(read(p, ['x', 'identifier'], ['to', 'keyword'])).toBeNull();
+  });
+});

package/src/core/matching/stock-reader.ts

@@ -0,0 +1,186 @@
+/**
+ * The stock value reader: what a domain grammar's matcher reads at each role
+ * slot, on the matcher's skeleton (domains rethink Phase 4.7,
+ * `docs-internal/DOMAINS_RETHINK.md`). Hyperscript's reader is semantic's
+ * `HyperscriptValueReader`; a domain takes this one.
+ *
+ * - A **greedy** slot (`RoleSpec.greedy`: a SQL condition, a column list) reads
+ *   every token up to the next marker the pattern still owes, or the end, as
+ *   one expression (`readGreedy`).
+ * - Any other slot reads one value (`readRoleValue`, the old matcher's reading):
+ *   a selector, a quoted string, a number, a boolean, a duration, a URL, an
+ *   identifier, or one of the multi-token forms it knows (`user.name`,
+ *   `#dialog.showModal()`), checked against the slot's expected types.
+ * - A **required** slot reads nothing where the stream is at the marker the
+ *   pattern wants next (the old matcher's guard against an SOV marker taken as
+ *   a value). Where a required slot finds nothing to read, and the optional,
+ *   marker-less slot just before it read a value with nothing read since, that
+ *   value is given back and the required slot reads again (the old matcher's
+ *   backtrack: behaviorspec `when clicks`, where the optional actor took
+ *   `clicks`; flow `orders として 取得変数`, where the optional patient took
+ *   `orders`).
+ * - An **optional** slot may be skipped for a token the pattern wants next, the
+ *   skip kept only when the rest of the pattern then reads to the end.
+ */
+
+import type {
+  StreamMark,
+  LanguagePattern,
+  LanguageToken,
+  PatternToken,
+  SemanticRole,
+  SemanticValue,
+  TokenStream,
+} from '../types';
+import type { MatchView, RoleSlot, ValueReader } from './value-reader';
+import { readRoleValue, type ReadingProfile } from './stock-reading';
+
+/**
+ * The markers a greedy value stops at, from the pattern tokens still owed after
+ * it: the next literal (with its alternatives), and each optional group's
+ * leading literal up to and including the first required token.
+ */
+export function greedyStopMarkers(rest: readonly PatternToken[]): Set<string> {
+  const markers = new Set<string>();
+  const add = (pt: PatternToken & { type: 'literal' }): void => {
+    markers.add(pt.value.toLowerCase());
+    for (const alt of pt.alternatives ?? []) markers.add(alt.toLowerCase());
+  };
+  for (const pt of rest) {
+    if (pt.type === 'literal') {
+      add(pt);
+      break;
+    }
+    if (pt.type === 'group') {
+      const lead = pt.tokens.find(t => t.type === 'literal');
+      if (lead?.type === 'literal') add(lead);
+      // For optional groups, continue scanning past them so the greedy capture
+      // also sees subsequent required literals (the SOV verb after an optional
+      // WHERE group).
+      if (!pt.optional) break;
+    }
+  }
+  return markers;
+}
+
+/** Whether `token` is one of the stop markers, by its value or its normalized form. */
+function isStopMarker(token: LanguageToken, stopMarkers: ReadonlySet<string>): boolean {
+  if (stopMarkers.size === 0) return false;
+  if (stopMarkers.has(token.value.toLowerCase())) return true;
+  return !!token.normalized && stopMarkers.has(token.normalized.toLowerCase());
+}
+
+/**
+ * Read every token up to a stop marker (or the end) as one expression value,
+ * advancing past them; null (consuming nothing) when the first token stops it.
+ */
+export function readGreedy(
+  tokens: TokenStream,
+  stopMarkers: ReadonlySet<string>
+): SemanticValue | null {
+  const values: string[] = [];
+  while (!tokens.isAtEnd()) {
+    const next = tokens.peek();
+    if (!next || isStopMarker(next, stopMarkers)) break;
+    values.push(next.value);
+    tokens.advance();
+  }
+  return values.length > 0 ? { type: 'expression', raw: values.join(' ') } : null;
+}
+
+/** The stock reader, for a matcher whose profiles it may consult (possessive words). */
+export class StockValueReader implements ValueReader {
+  private profile: ReadingProfile | undefined;
+  /** The last optional, marker-less slot that read a value: what a failing required slot may take back. */
+  private lastOptional: { role: SemanticRole; start: StreamMark; end: number } | undefined;
+
+  constructor(
+    private readonly view: MatchView,
+    private readonly profiles: Readonly<Record<string, ReadingProfile>> = {}
+  ) {}
+
+  beginPattern(pattern: LanguagePattern): void {
+    this.profile = this.profiles[pattern.language] ?? { code: pattern.language };
+    this.lastOptional = undefined;
+  }
+
+  readRole(
+    tokens: TokenStream,
+    slot: RoleSlot,
+    captured: Map<SemanticRole, SemanticValue>,
+    next: PatternToken | undefined
+  ): boolean {
+    const start = tokens.mark();
+    if (this.readOnce(tokens, slot, captured, next, true)) {
+      if (!slot.optional) {
+        this.lastOptional = undefined;
+      } else if (
+        captured.has(slot.role) &&
+        tokens.position() > start.position &&
+        !this.view.slotAfterLiteral() &&
+        next?.type !== 'literal'
+      ) {
+        this.lastOptional = { role: slot.role, start, end: tokens.position() };
+      }
+      return true;
+    }
+    const before = this.lastOptional;
+    if (slot.optional || !before || before.end !== tokens.position()) return false;
+    // Give the optional slot's value back and read this one from there.
+    this.lastOptional = undefined;
+    tokens.reset(before.start);
+    captured.delete(before.role);
+    return this.readOnce(tokens, slot, captured, next, false);
+  }
+
+  /** One read of `slot`; `guard` refuses a required slot the marker the pattern wants next. */
+  private readOnce(
+    tokens: TokenStream,
+    slot: RoleSlot,
+    captured: Map<SemanticRole, SemanticValue>,
+    next: PatternToken | undefined,
+    guard: boolean
+  ): boolean {
+    // A required slot never takes the marker the pattern wants next as its
+    // value (`v0x を 実行`: `を` is the marker, not the patient).
+    if (guard && !slot.optional && next?.type === 'literal') {
+      const token = tokens.peek();
+      if (token && this.view.wouldMatch(next, token)) return false;
+    }
+    if (slot.greedy) {
+      const value = readGreedy(tokens, greedyStopMarkers(this.view.patternRest()));
+      if (value) {
+        captured.set(slot.role, value);
+        return true;
+      }
+      return slot.optional ?? false;
+    }
+    return readRoleValue(tokens, slot, captured, this.profile);
+  }
+
+  maySkipSlot(
+    patternToken: PatternToken,
+    token: LanguageToken | null,
+    next: PatternToken | undefined
+  ): boolean {
+    const slot =
+      patternToken.type === 'group' && patternToken.optional
+        ? patternToken.tokens.every(t => t.type === 'role')
+          ? patternToken.tokens[0]
+          : undefined
+        : patternToken;
+    if (!slot || slot.type !== 'role' || !slot.optional) return false;
+    return !!token && this.view.wouldMatch(next, token);
+  }
+
+  atClauseEnd(tokens: TokenStream): boolean {
+    return tokens.isAtEnd();
+  }
+}
+
+/** A reader factory for `MatchSkeleton`, reading with the given profiles. */
+export function createStockReader(
+  profiles: Readonly<Record<string, ReadingProfile>> = {}
+): (view: MatchView) => ValueReader {
+  return view => new StockValueReader(view, profiles);
+}