@higher.archi/boe 1.0.25 → 1.0.27

package/src/engines/sentiment/strategy.ts

@@ -0,0 +1,513 @@
+/**
+ * Sentiment Engine Strategy
+ *
+ * Core execution logic for all sentiment strategies:
+ * - token-level: Score individual tokens in the full text
+ * - document-level: Split into sentences, score each, average compounds
+ * - aspect-based: Match sentences to aspect terms, score per aspect
+ */
+
+import type { IWorkingMemory, Fact } from '../../core';
+
+import type {
+  CompiledSentimentRuleSet,
+  CompiledTokenLevelSentimentRuleSet,
+  CompiledDocumentLevelSentimentRuleSet,
+  CompiledAspectBasedSentimentRuleSet,
+  CompiledSentimentLexicon,
+  SentimentOptions,
+  SentimentResult,
+  SentenceResult,
+  AspectResult,
+  TokenScore,
+  TokenModifier
+} from './types';
+import { resolveSentimentLabel } from './types';
+
+// ========================================
+// Helpers
+// ========================================
+
+/** Split text into tokens on whitespace and punctuation boundaries */
+function tokenize(text: string): string[] {
+  return text.match(/\w+(?:'\w+)?/g) ?? [];
+}
+
+/** Check if a token is ALL CAPS (must be >= 2 chars) */
+function isAllCaps(token: string): boolean {
+  return token.length >= 2 && token === token.toUpperCase() && /[A-Z]/.test(token);
+}
+
+/** Count trailing exclamation marks in text */
+function countTrailingExclamations(text: string): number {
+  const match = text.match(/!+\s*$/);
+  return match ? match[0].replace(/\s/g, '').length : 0;
+}
+
+/** Split text into sentences using a regex pattern */
+function splitSentences(text: string, pattern: RegExp): string[] {
+  return text
+    .split(pattern)
+    .map(s => s.trim())
+    .filter(s => s.length > 0);
+}
+
+/** Sliding window multi-word idiom matching */
+function scanIdioms(
+  tokens: string[],
+  idioms: Record<string, number>,
+  maxLen: number
+): { startIdx: number; endIdx: number; phrase: string; score: number }[] {
+  if (maxLen === 0) return [];
+
+  const matches: { startIdx: number; endIdx: number; phrase: string; score: number }[] = [];
+  const lowerTokens = tokens.map(t => t.toLowerCase());
+
+  for (let i = 0; i < lowerTokens.length; i++) {
+    // Try longest match first
+    for (let len = Math.min(maxLen, lowerTokens.length - i); len >= 2; len--) {
+      const phrase = lowerTokens.slice(i, i + len).join(' ');
+      if (phrase in idioms) {
+        matches.push({
+          startIdx: i,
+          endIdx: i + len - 1,
+          phrase,
+          score: idioms[phrase]
+        });
+        break; // longest match wins for this position
+      }
+    }
+  }
+
+  return matches;
+}
+
+/** Core token scoring loop */
+function scoreTokens(
+  tokens: string[],
+  lexicon: CompiledSentimentLexicon,
+  capsBoost: number,
+  negationWindow: number,
+  onToken?: (token: TokenScore) => void
+): TokenScore[] {
+  const results: TokenScore[] = [];
+
+  // Pre-scan idioms
+  const idiomMatches = scanIdioms(tokens, lexicon.idioms, lexicon._idiomMaxLength);
+  const idiomCovered = new Set<number>();
+  for (const match of idiomMatches) {
+    for (let i = match.startIdx; i <= match.endIdx; i++) {
+      idiomCovered.add(i);
+    }
+  }
+
+  // Insert idiom scores at their start positions
+  const idiomAtStart = new Map<number, { phrase: string; score: number }>();
+  for (const match of idiomMatches) {
+    idiomAtStart.set(match.startIdx, { phrase: match.phrase, score: match.score });
+  }
+
+  let negationCountdown = 0;
+  let pendingBooster = 0;
+
+  for (let i = 0; i < tokens.length; i++) {
+    const token = tokens[i];
+    const normalized = token.toLowerCase();
+
+    // Check if this token starts an idiom
+    if (idiomAtStart.has(i)) {
+      const idiom = idiomAtStart.get(i)!;
+      let adjustedScore = idiom.score;
+      const modifiers: TokenModifier[] = ['idiom'];
+
+      if (negationCountdown > 0) {
+        adjustedScore *= -1;
+        modifiers.push('negated');
+        negationCountdown--;
+      }
+
+      if (pendingBooster !== 0) {
+        adjustedScore *= pendingBooster;
+        modifiers.push('boosted');
+        pendingBooster = 0;
+      }
+
+      const result: TokenScore = {
+        token: idiom.phrase,
+        rawScore: idiom.score,
+        adjustedScore,
+        modifiers
+      };
+      results.push(result);
+      if (onToken) onToken(result);
+      continue;
+    }
+
+    // Skip tokens covered by an idiom (not at start position)
+    if (idiomCovered.has(i)) continue;
+
+    // Check negation
+    if (lexicon.negation.has(normalized)) {
+      negationCountdown = negationWindow;
+      const result: TokenScore = {
+        token,
+        rawScore: 0,
+        adjustedScore: 0,
+        modifiers: []
+      };
+      results.push(result);
+      if (onToken) onToken(result);
+      continue;
+    }
+
+    // Check booster
+    if (normalized in lexicon.boosters) {
+      pendingBooster = lexicon.boosters[normalized];
+      const result: TokenScore = {
+        token,
+        rawScore: 0,
+        adjustedScore: 0,
+        modifiers: []
+      };
+      results.push(result);
+      if (onToken) onToken(result);
+      continue;
+    }
+
+    // Lexicon lookup
+    const rawScore = lexicon.lexicon[normalized] ?? 0;
+
+    if (rawScore === 0) {
+      const result: TokenScore = {
+        token,
+        rawScore: 0,
+        adjustedScore: 0,
+        modifiers: []
+      };
+      results.push(result);
+      if (onToken) onToken(result);
+      continue;
+    }
+
+    let adjustedScore = rawScore;
+    const modifiers: TokenModifier[] = [];
+
+    // Apply pending booster
+    if (pendingBooster !== 0) {
+      adjustedScore *= pendingBooster;
+      modifiers.push('boosted');
+      pendingBooster = 0;
+    }
+
+    // Apply negation
+    if (negationCountdown > 0) {
+      adjustedScore *= -1;
+      modifiers.push('negated');
+      negationCountdown--;
+    }
+
+    // Apply caps boost
+    if (isAllCaps(token)) {
+      adjustedScore *= capsBoost;
+      modifiers.push('caps');
+    }
+
+    const result: TokenScore = {
+      token,
+      rawScore,
+      adjustedScore,
+      modifiers
+    };
+    results.push(result);
+    if (onToken) onToken(result);
+  }
+
+  return results;
+}
+
+/** Compute compound score: normalize via sum / sqrt(sum^2 + alpha), apply exclamation boost */
+function computeCompound(
+  tokenScores: TokenScore[],
+  exclamationBoost: number,
+  exclamationCount: number
+): number {
+  const alpha = 15;
+  const sum = tokenScores.reduce((acc, t) => acc + t.adjustedScore, 0);
+  let compound = sum / Math.sqrt(sum * sum + alpha);
+
+  // Apply exclamation boost
+  if (exclamationCount > 0) {
+    const boost = Math.pow(exclamationBoost, Math.min(exclamationCount, 4));
+    compound = compound * boost;
+  }
+
+  // Clamp to [-1, 1]
+  return Math.max(-1, Math.min(1, compound));
+}
+
+/** Compute positive/negative/neutral proportions from token scores */
+function computeProportions(tokenScores: TokenScore[]): { positive: number; negative: number; neutral: number } {
+  if (tokenScores.length === 0) {
+    return { positive: 0, negative: 0, neutral: 1 };
+  }
+
+  let pos = 0;
+  let neg = 0;
+  let neu = 0;
+
+  for (const t of tokenScores) {
+    if (t.adjustedScore > 0) pos++;
+    else if (t.adjustedScore < 0) neg++;
+    else neu++;
+  }
+
+  const total = tokenScores.length;
+  return {
+    positive: round(pos / total, 4),
+    negative: round(neg / total, 4),
+    neutral: round(neu / total, 4)
+  };
+}
+
+function round(value: number, decimals: number): number {
+  const factor = Math.pow(10, decimals);
+  return Math.round(value * factor) / factor;
+}
+
+// ========================================
+// Executor
+// ========================================
+
+export class SentimentExecutor {
+  run(
+    ruleSet: CompiledSentimentRuleSet,
+    wm: IWorkingMemory,
+    options: SentimentOptions = {}
+  ): SentimentResult {
+    const startTime = performance.now();
+
+    // Gather all text facts from working memory
+    const facts = wm.getAll();
+    const texts = extractTexts(facts);
+    const fullText = texts.join(' ');
+
+    return this.scoreText(ruleSet, fullText, options, startTime);
+  }
+
+  /** Score a single text string directly (no WorkingMemory involved) */
+  scoreText(
+    ruleSet: CompiledSentimentRuleSet,
+    text: string,
+    options: SentimentOptions = {},
+    startTime: number = performance.now()
+  ): SentimentResult {
+    switch (ruleSet.strategy) {
+      case 'token-level':
+        return this.runTokenLevel(ruleSet, text, options, startTime);
+      case 'document-level':
+        return this.runDocumentLevel(ruleSet, text, options, startTime);
+      case 'aspect-based':
+        return this.runAspectBased(ruleSet, text, options, startTime);
+      default:
+        throw new Error(`Unknown sentiment strategy: '${(ruleSet as any).strategy}'`);
+    }
+  }
+
+  // ========================================
+  // Token-Level Strategy
+  // ========================================
+
+  private runTokenLevel(
+    ruleSet: CompiledTokenLevelSentimentRuleSet,
+    text: string,
+    options: SentimentOptions,
+    startTime: number
+  ): SentimentResult {
+    const tokens = tokenize(text);
+    const exclamationCount = countTrailingExclamations(text);
+    const tokenScores = scoreTokens(
+      tokens, ruleSet.lexicon, ruleSet.config.capsBoost,
+      ruleSet.config.negationWindow, options.onToken
+    );
+
+    const compound = computeCompound(tokenScores, ruleSet.config.exclamationBoost, exclamationCount);
+    const proportions = computeProportions(tokenScores);
+    const executionTimeMs = round((performance.now() - startTime) * 100, 0) / 100;
+
+    return {
+      compound: round(compound, 4),
+      label: resolveSentimentLabel(compound),
+      positive: proportions.positive,
+      negative: proportions.negative,
+      neutral: proportions.neutral,
+      tokens: tokenScores,
+      strategy: 'token-level',
+      executionTimeMs
+    };
+  }
+
+  // ========================================
+  // Document-Level Strategy
+  // ========================================
+
+  private runDocumentLevel(
+    ruleSet: CompiledDocumentLevelSentimentRuleSet,
+    text: string,
+    options: SentimentOptions,
+    startTime: number
+  ): SentimentResult {
+    const sentences = splitSentences(text, ruleSet.sentenceSplitPattern);
+
+    if (sentences.length === 0) {
+      const executionTimeMs = round((performance.now() - startTime) * 100, 0) / 100;
+      return {
+        compound: 0,
+        label: 'neutral',
+        positive: 0,
+        negative: 0,
+        neutral: 1,
+        sentences: [],
+        strategy: 'document-level',
+        executionTimeMs
+      };
+    }
+
+    const sentenceResults: SentenceResult[] = [];
+
+    for (const sentenceText of sentences) {
+      const tokens = tokenize(sentenceText);
+      const exclamationCount = countTrailingExclamations(sentenceText);
+      const tokenScores = scoreTokens(
+        tokens, ruleSet.lexicon, ruleSet.config.capsBoost,
+        ruleSet.config.negationWindow, options.onToken
+      );
+      const compound = computeCompound(tokenScores, ruleSet.config.exclamationBoost, exclamationCount);
+
+      sentenceResults.push({
+        text: sentenceText,
+        compound: round(compound, 4),
+        label: resolveSentimentLabel(compound),
+        tokens: tokenScores
+      });
+    }
+
+    // Overall compound is average of sentence compounds
+    const avgCompound = sentenceResults.reduce((sum, s) => sum + s.compound, 0) / sentenceResults.length;
+
+    // Aggregate proportions across all tokens
+    const allTokens = sentenceResults.flatMap(s => s.tokens);
+    const proportions = computeProportions(allTokens);
+    const executionTimeMs = round((performance.now() - startTime) * 100, 0) / 100;
+
+    return {
+      compound: round(avgCompound, 4),
+      label: resolveSentimentLabel(avgCompound),
+      positive: proportions.positive,
+      negative: proportions.negative,
+      neutral: proportions.neutral,
+      sentences: sentenceResults,
+      strategy: 'document-level',
+      executionTimeMs
+    };
+  }
+
+  // ========================================
+  // Aspect-Based Strategy
+  // ========================================
+
+  private runAspectBased(
+    ruleSet: CompiledAspectBasedSentimentRuleSet,
+    text: string,
+    options: SentimentOptions,
+    startTime: number
+  ): SentimentResult {
+    // Split into sentences using default pattern
+    const sentences = splitSentences(text, /[.!?]+/);
+
+    const aspectResults: AspectResult[] = [];
+
+    for (const aspect of ruleSet.aspects) {
+      const matchingSentences: SentenceResult[] = [];
+
+      for (const sentenceText of sentences) {
+        // Check if sentence contains the aspect term
+        if (!sentenceText.toLowerCase().includes(aspect)) continue;
+
+        const tokens = tokenize(sentenceText);
+        const exclamationCount = countTrailingExclamations(sentenceText);
+        const tokenScores = scoreTokens(
+          tokens, ruleSet.lexicon, ruleSet.config.capsBoost,
+          ruleSet.config.negationWindow, options.onToken
+        );
+        const compound = computeCompound(tokenScores, ruleSet.config.exclamationBoost, exclamationCount);
+
+        matchingSentences.push({
+          text: sentenceText,
+          compound: round(compound, 4),
+          label: resolveSentimentLabel(compound),
+          tokens: tokenScores
+        });
+      }
+
+      const aspectCompound = matchingSentences.length > 0
+        ? matchingSentences.reduce((sum, s) => sum + s.compound, 0) / matchingSentences.length
+        : 0;
+
+      aspectResults.push({
+        aspect,
+        compound: round(aspectCompound, 4),
+        label: resolveSentimentLabel(aspectCompound),
+        sentences: matchingSentences
+      });
+    }
+
+    // Overall compound is average of aspect compounds (only aspects with data)
+    const aspectsWithData = aspectResults.filter(a => a.sentences.length > 0);
+    const avgCompound = aspectsWithData.length > 0
+      ? aspectsWithData.reduce((sum, a) => sum + a.compound, 0) / aspectsWithData.length
+      : 0;
+
+    // Aggregate proportions from all aspect tokens
+    const allTokens = aspectResults.flatMap(a => a.sentences.flatMap(s => s.tokens));
+    const proportions = computeProportions(allTokens);
+    const executionTimeMs = round((performance.now() - startTime) * 100, 0) / 100;
+
+    return {
+      compound: round(avgCompound, 4),
+      label: resolveSentimentLabel(avgCompound),
+      positive: proportions.positive,
+      negative: proportions.negative,
+      neutral: proportions.neutral,
+      aspects: aspectResults,
+      strategy: 'aspect-based',
+      executionTimeMs
+    };
+  }
+}
+
+// ========================================
+// Module-Level Helpers
+// ========================================
+
+/** Extract text content from facts - looks for text/content/body/message string fields */
+function extractTexts(facts: Fact[]): string[] {
+  const texts: string[] = [];
+  for (const fact of facts) {
+    const data = fact.data;
+    if (typeof data === 'string') {
+      texts.push(data);
+    } else if (data && typeof data === 'object') {
+      // Try common text field names
+      for (const key of ['text', 'content', 'body', 'message', 'review', 'comment', 'feedback']) {
+        if (typeof (data as any)[key] === 'string') {
+          texts.push((data as any)[key]);
+          break;
+        }
+      }
+    }
+  }
+  return texts;
+}
+
+/** Singleton instance */
+export const sentimentStrategy = new SentimentExecutor();

package/src/engines/sentiment/types.ts

@@ -0,0 +1,198 @@
+/**
+ * Sentiment Engine Types
+ *
+ * Text sentiment analysis engine that scores text using dictionary-lookup
+ * and heuristic rules. Supports token-level, document-level, and aspect-based
+ * analysis strategies with injectable lexicon configuration.
+ */
+
+// ========================================
+// Semantic Types
+// ========================================
+
+/** Sentiment analysis strategy */
+export type SentimentStrategy = 'token-level' | 'document-level' | 'aspect-based';
+
+/** Resolved sentiment label */
+export type SentimentLabel = 'positive' | 'neutral' | 'negative' | 'mixed';
+
+// ========================================
+// Lexicon Types
+// ========================================
+
+/** Injectable lexicon configuration (user-facing, optional fields) */
+export type SentimentLexicon = {
+  lexicon: Record<string, number>;
+  boosters?: Record<string, number>;
+  negation?: string[];
+  idioms?: Record<string, number>;
+};
+
+/** Compiled lexicon with all fields required, keys lowercased */
+export type CompiledSentimentLexicon = {
+  lexicon: Record<string, number>;
+  boosters: Record<string, number>;
+  negation: Set<string>;
+  idioms: Record<string, number>;
+  _idiomMaxLength: number;
+};
+
+// ========================================
+// Config Defaults
+// ========================================
+
+export const SENTIMENT_DEFAULTS = {
+  capsBoost: 1.2,
+  exclamationBoost: 1.1,
+  negationWindow: 3
+} as const;
+
+// ========================================
+// Strategy-Specific Config Types
+// ========================================
+
+export type SentimentConfig = {
+  capsBoost?: number;
+  exclamationBoost?: number;
+  negationWindow?: number;
+};
+
+// ========================================
+// Source RuleSet Types (Discriminated Union)
+// ========================================
+
+type SentimentRuleSetBase = {
+  id: string;
+  name?: string;
+  mode: 'sentiment';
+  lexiconConfig: SentimentLexicon;
+  config?: SentimentConfig;
+};
+
+/** Token-level: score individual tokens in the full text */
+export type TokenLevelSentimentRuleSet = SentimentRuleSetBase & {
+  strategy: 'token-level';
+};
+
+/** Document-level: split into sentences, score each, average */
+export type DocumentLevelSentimentRuleSet = SentimentRuleSetBase & {
+  strategy: 'document-level';
+  sentenceSplitPattern?: string;
+};
+
+/** Aspect-based: match sentences to aspect terms, score per aspect */
+export type AspectBasedSentimentRuleSet = SentimentRuleSetBase & {
+  strategy: 'aspect-based';
+  aspects: string[];
+};
+
+export type SentimentRuleSet =
+  | TokenLevelSentimentRuleSet
+  | DocumentLevelSentimentRuleSet
+  | AspectBasedSentimentRuleSet;
+
+// ========================================
+// Compiled RuleSet Types
+// ========================================
+
+type CompiledSentimentRuleSetBase = {
+  id: string;
+  name?: string;
+  mode: 'sentiment';
+  lexicon: CompiledSentimentLexicon;
+  config: {
+    capsBoost: number;
+    exclamationBoost: number;
+    negationWindow: number;
+  };
+};
+
+export type CompiledTokenLevelSentimentRuleSet = CompiledSentimentRuleSetBase & {
+  strategy: 'token-level';
+};
+
+export type CompiledDocumentLevelSentimentRuleSet = CompiledSentimentRuleSetBase & {
+  strategy: 'document-level';
+  sentenceSplitPattern: RegExp;
+};
+
+export type CompiledAspectBasedSentimentRuleSet = CompiledSentimentRuleSetBase & {
+  strategy: 'aspect-based';
+  aspects: string[];
+};
+
+export type CompiledSentimentRuleSet =
+  | CompiledTokenLevelSentimentRuleSet
+  | CompiledDocumentLevelSentimentRuleSet
+  | CompiledAspectBasedSentimentRuleSet;
+
+// ========================================
+// Runtime Result Types
+// ========================================
+
+/** Modifier applied to a token score */
+export type TokenModifier = 'negated' | 'boosted' | 'caps' | 'idiom';
+
+/** Per-token scoring detail */
+export type TokenScore = {
+  token: string;
+  rawScore: number;
+  adjustedScore: number;
+  modifiers: TokenModifier[];
+};
+
+/** Per-sentence result (document-level and aspect-based) */
+export type SentenceResult = {
+  text: string;
+  compound: number;
+  label: SentimentLabel;
+  tokens: TokenScore[];
+};
+
+/** Per-aspect result (aspect-based only) */
+export type AspectResult = {
+  aspect: string;
+  compound: number;
+  label: SentimentLabel;
+  sentences: SentenceResult[];
+};
+
+/** Full sentiment analysis result */
+export type SentimentResult = {
+  compound: number;
+  label: SentimentLabel;
+  positive: number;
+  negative: number;
+  neutral: number;
+  tokens?: TokenScore[];
+  sentences?: SentenceResult[];
+  aspects?: AspectResult[];
+  strategy: SentimentStrategy;
+  executionTimeMs: number;
+};
+
+/** Runtime options */
+export type SentimentOptions = {
+  onToken?: (token: TokenScore) => void;
+};
+
+/** Result from ingest() -- includes both the individual text score and the running aggregate */
+export type IngestResult = SentimentResult & {
+  /** Compound score for just the ingested text (not the aggregate) */
+  itemCompound: number;
+  /** Label for just the ingested text */
+  itemLabel: SentimentLabel;
+  /** Total number of texts ingested so far */
+  totalIngested: number;
+};
+
+// ========================================
+// Resolver Functions
+// ========================================
+
+/** Resolve a compound score (-1 to 1) to a sentiment label */
+export function resolveSentimentLabel(compound: number): SentimentLabel {
+  if (compound > 0.05) return 'positive';
+  if (compound < -0.05) return 'negative';
+  return 'neutral';
+}