@onmars/lunar-core 0.1.0

package/src/lib/daemon.ts

@@ -0,0 +1,216 @@
+import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'
+import { dirname, resolve } from 'node:path'
+import type { MemoryProvider } from '../types/memory'
+import type { ChannelPersona } from '../types/moon'
+import type { LunarConfig } from './config'
+import type { MemoryConfig, RecallConfig, SecurityConfig, VoiceConfig } from './config-loader'
+import { log } from './logger'
+import type { MoonLoader } from './moon-loader'
+import type { PluginRegistry } from './plugin'
+import { Router } from './router'
+import type { Scheduler } from './scheduler'
+import type { SessionStore } from './session'
+
+export interface DaemonOptions {
+  config: LunarConfig
+  registry: PluginRegistry
+  sessions: SessionStore
+  moonLoader: MoonLoader
+  channels: ChannelPersona[]
+  security?: SecurityConfig
+  /** Memory provider for auto-recall */
+  memory?: MemoryProvider
+  /** Auto-recall configuration */
+  /** Recall configuration (enabled, limit, minScore, budget) */
+  recallConfig?: RecallConfig
+  /** Workspace path (for loading prompts/) */
+  workspacePath?: string
+  /** Full memory config (for provider name, instructionsFile) */
+  memoryConfig?: MemoryConfig
+  /** Scheduler instance (optional — created by CLI start if config exists) */
+  scheduler?: Scheduler
+  /** Voice configuration (TTS mode, STT mode) */
+  voiceConfig?: VoiceConfig
+}
+
+/**
+ * Daemon — The main process lifecycle manager.
+ *
+ * Handles: PID lock, signal handlers, graceful shutdown, startup orchestration.
+ */
+export class Daemon {
+  private pidPath: string
+  private router: Router
+  private shutdownPromise?: Promise<void>
+  private startedAt = 0
+  private scheduler?: Scheduler
+
+  constructor(private options: DaemonOptions) {
+    this.pidPath = resolve(options.config.dataDir, 'lunar.pid')
+    this.scheduler = options.scheduler
+    this.router = new Router(
+      options.registry,
+      options.sessions,
+      options.config,
+      options.moonLoader,
+      options.channels,
+      options.security,
+      options.memory,
+      options.recallConfig,
+      options.workspacePath,
+      options.memoryConfig,
+      options.scheduler,
+      options.voiceConfig,
+    )
+  }
+
+  /** Start the daemon */
+  async start(): Promise<void> {
+    // Acquire PID lock
+    this.acquirePidLock()
+
+    // Register signal handlers
+    this.registerSignals()
+
+    this.startedAt = Date.now()
+    this.router.daemonStartedAt = this.startedAt
+
+    log.info({ pid: process.pid, dev: this.options.config.dev }, '🌑 Lunar starting')
+
+    // Initialize all plugins
+    await this.options.registry.initAll()
+
+    // Wire up message routing
+    this.wireRouting()
+
+    // Start scheduler if configured
+    if (this.scheduler) {
+      await this.scheduler.start()
+    }
+
+    log.info(
+      {
+        channels: this.options.registry.listChannels(),
+        agents: this.options.registry.listAgents(),
+        memory: this.options.registry.listMemory(),
+        moons: this.options.moonLoader.list(),
+        scheduler: !!this.scheduler,
+      },
+      '🌑 Lunar ready',
+    )
+  }
+
+  /** Graceful shutdown */
+  async stop(): Promise<void> {
+    if (this.shutdownPromise) return this.shutdownPromise
+
+    this.shutdownPromise = (async () => {
+      log.info('🌑 Lunar shutting down...')
+
+      // Stop scheduler first (no new dispatches during shutdown)
+      if (this.scheduler) {
+        await this.scheduler.stop()
+      }
+
+      // Graceful session close — triggers memory lifecycle hooks
+      // (summarize, promote, etc.) before providers are destroyed
+      try {
+        await this.router.closeAllSessions()
+      } catch (err) {
+        log.warn({ err }, 'Error during session close — continuing shutdown')
+      }
+
+      // Close sessions DB
+      this.options.sessions.close()
+
+      // Destroy all plugins
+      await this.options.registry.destroyAll()
+
+      // Release PID lock
+      this.releasePidLock()
+
+      log.info('🌑 Lunar stopped')
+    })()
+
+    return this.shutdownPromise
+  }
+
+  /** Get the router instance */
+  getRouter(): Router {
+    return this.router
+  }
+
+  // --- Private ---
+
+  private wireRouting(): void {
+    const { registry } = this.options
+
+    for (const channelId of registry.listChannels()) {
+      const channel = registry.getChannel(channelId)
+      if (!channel) continue
+
+      channel.onMessage(async (msg) => {
+        await this.router.route(channelId, msg)
+      })
+
+      log.debug({ channel: channelId }, 'Message routing wired')
+    }
+  }
+
+  private acquirePidLock(): void {
+    const dir = dirname(this.pidPath)
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true })
+
+    // Check for stale PID
+    if (existsSync(this.pidPath)) {
+      const existingPid = readFileSync(this.pidPath, 'utf-8').trim()
+      try {
+        // Check if process is still running
+        process.kill(Number.parseInt(existingPid, 10), 0)
+        throw new Error(`Lunar is already running (PID ${existingPid})`)
+      } catch (err: unknown) {
+        if (
+          err instanceof Error &&
+          'code' in err &&
+          (err as NodeJS.ErrnoException).code === 'ESRCH'
+        ) {
+          log.warn({ stalePid: existingPid }, 'Removing stale PID file')
+          unlinkSync(this.pidPath)
+        } else {
+          throw err
+        }
+      }
+    }
+
+    writeFileSync(this.pidPath, String(process.pid))
+    log.debug({ pid: process.pid, path: this.pidPath }, 'PID lock acquired')
+  }
+
+  private releasePidLock(): void {
+    try {
+      if (existsSync(this.pidPath)) {
+        unlinkSync(this.pidPath)
+      }
+    } catch {
+      // Best effort
+    }
+  }
+
+  private registerSignals(): void {
+    const shutdown = async (signal: string) => {
+      log.info({ signal }, 'Received signal')
+      await this.stop()
+      process.exit(0)
+    }
+
+    process.on('SIGINT', () => shutdown('SIGINT'))
+    process.on('SIGTERM', () => shutdown('SIGTERM'))
+    process.on('uncaughtException', (err) => {
+      log.fatal({ err }, 'Uncaught exception')
+      this.stop().finally(() => process.exit(1))
+    })
+    process.on('unhandledRejection', (err) => {
+      log.error({ err }, 'Unhandled rejection')
+    })
+  }
+}

package/src/lib/dedup.ts

@@ -0,0 +1,414 @@
+/**
+ * @module dedup — Near-duplicate detection for memory results
+ *
+ * Zero external dependencies. Three strategies:
+ *
+ * 1. **Dice (bigram)** — Character bigram overlap (Sørensen–Dice coefficient).
+ *    Best for short texts (<300 chars). O(n) time, O(n) space.
+ *
+ * 2. **MinHash** — Approximate Jaccard similarity via locality-sensitive hashing.
+ *    Uses word-level trigram shingling + FNV-1a hash + random permutations.
+ *    Best for medium-long texts. O(n·k) time where k = numPermutations.
+ *    Based on: Broder (1997), BigCode/HuggingFace dedup pipeline, GPT-3 paper.
+ *
+ * 3. **Adaptive** — Auto-selects Dice or MinHash based on text length.
+ *    Default strategy. Crossover at ~300 chars.
+ *
+ * Design priorities:
+ * - Conservative: high precision (don't wrongly remove unique content) over
+ *   high recall (catch every duplicate). Missing a duplicate = wasted tokens.
+ *   Removing unique content = lost context.
+ * - Zero dependencies: pure TypeScript, no npm packages.
+ * - Framework-ready: pluggable DedupStrategy interface for custom implementations.
+ *
+ * References:
+ * - Sørensen–Dice: https://en.wikipedia.org/wiki/Dice-S%C3%B8rensen_coefficient
+ * - MinHash: Broder (1997), "On the resemblance and containment of documents"
+ * - "In Defense of MinHash Over SimHash" (Shrivastava & Li, AISTATS 2014)
+ * - HuggingFace BigCode: word-level trigram shingling standard
+ * - FNV-1a: Fowler–Noll–Vo non-cryptographic hash
+ */
+
+// ════════════════════════════════════════════════════════════
+// Types
+// ════════════════════════════════════════════════════════════
+
+export interface DedupResult<T> {
+  /** Items kept after deduplication */
+  kept: T[]
+  /** Items removed as duplicates (with reference to what they duplicated) */
+  removed: Array<{ item: T; duplicateOf: T; similarity: number }>
+}
+
+export interface DedupStrategy {
+  /** Compare two texts. Returns 0–1 similarity score. */
+  compare(a: string, b: string): number
+  /** Strategy identifier for logging/debugging */
+  readonly id: string
+}
+
+export interface DedupOptions {
+  /** Similarity threshold above which items are considered duplicates. Default: 0.85 */
+  threshold?: number
+  /** Maximum number of items to keep (token budget proxy). Default: Infinity */
+  maxResults?: number
+  /** Extract text from item for comparison. Default: String(item) */
+  textExtractor?: (item: unknown) => string
+}
+
+// ════════════════════════════════════════════════════════════
+// Text normalization
+// ════════════════════════════════════════════════════════════
+
+const WHITESPACE_RE = /\s+/g
+const PUNCTUATION_RE = /[^\p{L}\p{N}\s]/gu
+
+/**
+ * Normalize text for comparison: lowercase, collapse whitespace,
+ * strip punctuation (keeping letters + numbers in any language).
+ */
+export function normalize(text: string): string {
+  return text.toLowerCase().replace(PUNCTUATION_RE, ' ').replace(WHITESPACE_RE, ' ').trim()
+}
+
+// ════════════════════════════════════════════════════════════
+// FNV-1a hash (32-bit)
+// ════════════════════════════════════════════════════════════
+
+const FNV_OFFSET = 0x811c9dc5
+const FNV_PRIME = 0x01000193
+
+/**
+ * FNV-1a 32-bit hash. Fast, well-distributed, non-cryptographic.
+ * Used internally by MinHash for shingling.
+ */
+export function fnv1a32(str: string): number {
+  let hash = FNV_OFFSET
+  for (let i = 0; i < str.length; i++) {
+    hash ^= str.charCodeAt(i)
+    hash = Math.imul(hash, FNV_PRIME) >>> 0
+  }
+  return hash
+}
+
+// ════════════════════════════════════════════════════════════
+// Dice coefficient (bigram-based)
+// ════════════════════════════════════════════════════════════
+
+/**
+ * Sørensen–Dice coefficient using character bigrams.
+ * Handles multiset (counts duplicates correctly: "GG" vs "GGGG" ≠ 1.0).
+ *
+ * Accuracy: ~85% precision on short texts, ~97%+ at threshold 0.90.
+ * Speed: O(n+m) time, O(n+m) space.
+ *
+ * @returns 0–1 similarity score
+ */
+export function diceSimilarity(a: string, b: string): number {
+  if (a === b) return 1
+  if (a.length < 2 || b.length < 2) return 0
+
+  const bigramsA = new Map<string, number>()
+  const bigramsB = new Map<string, number>()
+
+  for (let i = 0; i < a.length - 1; i++) {
+    const bg = a.charAt(i) + a.charAt(i + 1)
+    bigramsA.set(bg, (bigramsA.get(bg) || 0) + 1)
+  }
+  for (let i = 0; i < b.length - 1; i++) {
+    const bg = b.charAt(i) + b.charAt(i + 1)
+    bigramsB.set(bg, (bigramsB.get(bg) || 0) + 1)
+  }
+
+  let intersection = 0
+  for (const [bg, count] of bigramsA) {
+    const countB = bigramsB.get(bg)
+    if (countB) intersection += Math.min(count, countB)
+  }
+
+  return (2 * intersection) / (a.length - 1 + b.length - 1)
+}
+
+export class DiceDedupStrategy implements DedupStrategy {
+  readonly id = 'dice'
+
+  compare(a: string, b: string): number {
+    return diceSimilarity(normalize(a), normalize(b))
+  }
+}
+
+// ════════════════════════════════════════════════════════════
+// MinHash (approximate Jaccard via LSH)
+// ════════════════════════════════════════════════════════════
+
+/**
+ * Generate word-level n-gram shingles from text.
+ * Default: trigrams (n=3), the standard for text dedup (BigCode, GPT-3).
+ *
+ * "the quick brown fox" with n=3 → ["the quick brown", "quick brown fox"]
+ */
+export function shingle(text: string, n = 3): string[] {
+  const words = normalize(text)
+    .split(' ')
+    .filter((w) => w.length > 0)
+  if (words.length < n) return [words.join(' ')]
+
+  const shingles: string[] = []
+  for (let i = 0; i <= words.length - n; i++) {
+    shingles.push(words.slice(i, i + n).join(' '))
+  }
+  return shingles
+}
+
+// Mersenne prime 2^31 - 1 (standard for MinHash permutation simulation)
+const MERSENNE_PRIME = 0x7fffffff
+
+export interface PermutationCoeffs {
+  a: Uint32Array
+  b: Uint32Array
+  numPerm: number
+}
+
+/**
+ * Generate random permutation coefficients for MinHash.
+ * Deterministic given the same seed (for reproducibility).
+ *
+ * Uses xorshift32 PRNG seeded from the input.
+ * These coefficients are reusable across all comparisons.
+ */
+export function generatePermutations(numPerm = 128, seed = 42): PermutationCoeffs {
+  const a = new Uint32Array(numPerm)
+  const b = new Uint32Array(numPerm)
+
+  // Seeded PRNG (xorshift32)
+  let state = seed >>> 0 || 1
+  function next(): number {
+    state ^= state << 13
+    state ^= state >>> 17
+    state ^= state << 5
+    return state >>> 0
+  }
+
+  for (let i = 0; i < numPerm; i++) {
+    // a must be non-zero mod prime for good distribution
+    a[i] = (next() % (MERSENNE_PRIME - 1)) + 1
+    b[i] = next() % MERSENNE_PRIME
+  }
+
+  return { a, b, numPerm }
+}
+
+/**
+ * Compute a MinHash signature for a set of shingles.
+ *
+ * Uses the random permutation simulation technique:
+ *   h_i(x) = (a_i * hash(x) + b_i) mod prime
+ * where a_i, b_i are random coefficients per permutation.
+ *
+ * The signature is a Uint32Array of `numPerm` minimum hash values.
+ */
+export function computeMinHash(shingles: string[], coeffs: PermutationCoeffs): Uint32Array {
+  const { a, b, numPerm } = coeffs
+  const signature = new Uint32Array(numPerm).fill(0xffffffff)
+
+  for (const s of shingles) {
+    const hash = fnv1a32(s)
+    for (let i = 0; i < numPerm; i++) {
+      // Simulate permutation: (a[i] * hash + b[i]) mod MERSENNE_PRIME
+      const permuted = ((Math.imul(a[i], hash) >>> 0) + b[i]) % MERSENNE_PRIME
+      if (permuted < signature[i]) {
+        signature[i] = permuted
+      }
+    }
+  }
+
+  return signature
+}
+
+/**
+ * Estimate Jaccard similarity from two MinHash signatures.
+ * Fraction of matching positions ≈ Jaccard(A, B).
+ *
+ * Standard error: 1/√numPerm
+ * - 128 perms → SE ~0.088 (±8.8%)
+ * - 256 perms → SE ~0.063 (±6.3%)
+ */
+export function minhashSimilarity(sigA: Uint32Array, sigB: Uint32Array): number {
+  if (sigA.length !== sigB.length) {
+    throw new Error(`Signature length mismatch: ${sigA.length} vs ${sigB.length}`)
+  }
+  let matches = 0
+  for (let i = 0; i < sigA.length; i++) {
+    if (sigA[i] === sigB[i]) matches++
+  }
+  return matches / sigA.length
+}
+
+/** Exact Jaccard similarity for small sets (no approximation needed). */
+function jaccardExact(a: Set<string>, b: Set<string>): number {
+  let intersection = 0
+  for (const item of a) {
+    if (b.has(item)) intersection++
+  }
+  const union = a.size + b.size - intersection
+  return union === 0 ? 1 : intersection / union
+}
+
+export class MinHashDedupStrategy implements DedupStrategy {
+  readonly id = 'minhash'
+  private readonly coeffs: PermutationCoeffs
+  private readonly shingleSize: number
+
+  /**
+   * @param numPerm - Number of permutations (more = more accurate, slower).
+   *                  128 = standard error ~8.8%. 256 = ~6.25%.
+   * @param shingleSize - Word n-gram size. 3 = standard (BigCode/GPT-3).
+   * @param seed - PRNG seed for reproducible permutations.
+   */
+  constructor(numPerm = 128, shingleSize = 3, seed = 42) {
+    this.coeffs = generatePermutations(numPerm, seed)
+    this.shingleSize = shingleSize
+  }
+
+  compare(a: string, b: string): number {
+    const shinglesA = shingle(a, this.shingleSize)
+    const shinglesB = shingle(b, this.shingleSize)
+
+    // Edge case: very short texts with few shingles — fall back to exact Jaccard
+    if (shinglesA.length <= 3 || shinglesB.length <= 3) {
+      return jaccardExact(new Set(shinglesA), new Set(shinglesB))
+    }
+
+    const sigA = computeMinHash(shinglesA, this.coeffs)
+    const sigB = computeMinHash(shinglesB, this.coeffs)
+    return minhashSimilarity(sigA, sigB)
+  }
+}
+
+// ════════════════════════════════════════════════════════════
+// Adaptive strategy (auto-selects based on text length)
+// ════════════════════════════════════════════════════════════
+
+export class AdaptiveDedupStrategy implements DedupStrategy {
+  readonly id = 'adaptive'
+  private readonly dice = new DiceDedupStrategy()
+  private readonly minhash: MinHashDedupStrategy
+  private readonly crossoverLength: number
+
+  /**
+   * @param crossoverLength - Char count threshold to switch Dice → MinHash.
+   *                          Default: 300 (empirically, Dice accuracy degrades
+   *                          for longer texts where boilerplate dilutes bigrams).
+   * @param numPerm - MinHash permutations (passed through).
+   */
+  constructor(crossoverLength = 300, numPerm = 128) {
+    this.crossoverLength = crossoverLength
+    this.minhash = new MinHashDedupStrategy(numPerm)
+  }
+
+  compare(a: string, b: string): number {
+    const maxLen = Math.max(a.length, b.length)
+    if (maxLen <= this.crossoverLength) {
+      return this.dice.compare(a, b)
+    }
+    return this.minhash.compare(a, b)
+  }
+}
+
+// ════════════════════════════════════════════════════════════
+// Deduplication engine
+// ════════════════════════════════════════════════════════════
+
+/**
+ * Deduplicate an array of items using the given strategy.
+ *
+ * Items are compared pairwise. When a duplicate is found, the item appearing
+ * later in the array is removed (earlier items have priority — sort by
+ * relevance score before calling this).
+ *
+ * Complexity: O(n² · C) where C = cost of one comparison.
+ * For n ≤ 20 (typical memory recall), this is negligible.
+ *
+ * @param items - Items to deduplicate (pre-sorted by relevance, best first)
+ * @param strategy - Comparison strategy (default: adaptive)
+ * @param options - Threshold, maxResults, text extractor
+ * @returns Kept items + removed items with metadata
+ */
+export function deduplicate<T>(
+  items: T[],
+  strategy: DedupStrategy = new AdaptiveDedupStrategy(),
+  options: DedupOptions = {},
+): DedupResult<T> {
+  const {
+    threshold = 0.85,
+    maxResults = Infinity,
+    textExtractor = (item: unknown) => String(item),
+  } = options
+
+  const kept: T[] = []
+  const removed: DedupResult<T>['removed'] = []
+
+  for (const item of items) {
+    if (kept.length >= maxResults) break
+
+    const text = textExtractor(item)
+    let isDuplicate = false
+
+    for (const existing of kept) {
+      const existingText = textExtractor(existing)
+      const similarity = strategy.compare(text, existingText)
+
+      if (similarity > threshold) {
+        removed.push({ item, duplicateOf: existing, similarity })
+        isDuplicate = true
+        break
+      }
+    }
+
+    if (!isDuplicate) {
+      kept.push(item)
+    }
+  }
+
+  return { kept, removed }
+}
+
+// ════════════════════════════════════════════════════════════
+// Factory — create strategy from config
+// ════════════════════════════════════════════════════════════
+
+export interface DedupConfig {
+  /** Strategy name: 'dice' | 'minhash' | 'adaptive' (default) */
+  strategy?: 'dice' | 'minhash' | 'adaptive'
+  /** Similarity threshold for duplicate detection. Default: 0.85 */
+  threshold?: number
+  /** MinHash: number of permutations. Default: 128 */
+  numPermutations?: number
+  /** MinHash: word n-gram size. Default: 3 */
+  shingleSize?: number
+  /** Adaptive: crossover length in chars. Default: 300 */
+  crossoverLength?: number
+  /** Maximum results after dedup. Default: Infinity */
+  maxResults?: number
+}
+
+/** Create a DedupStrategy from config. */
+export function createStrategy(config: DedupConfig = {}): DedupStrategy {
+  const {
+    strategy = 'adaptive',
+    numPermutations = 128,
+    shingleSize = 3,
+    crossoverLength = 300,
+  } = config
+
+  switch (strategy) {
+    case 'dice':
+      return new DiceDedupStrategy()
+    case 'minhash':
+      return new MinHashDedupStrategy(numPermutations, shingleSize)
+    case 'adaptive':
+      return new AdaptiveDedupStrategy(crossoverLength, numPermutations)
+    default:
+      return new AdaptiveDedupStrategy(crossoverLength, numPermutations)
+  }
+}