@remnic/core 9.65.7 → 9.65.8

package/dist/chunk-ZPUIRTIZ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/lcm/summarizer.ts"],"sourcesContent":["import { log } from \"../logger.js\";\nimport { looksLikeMechanicalTelemetryTranscript } from \"../telemetry-transcript.js\";\nimport { estimateTokenCount } from \"../token-estimate.js\";\nimport type { LcmArchive, LcmMessage } from \"./archive.js\";\nimport { estimateTokens } from \"./archive.js\";\nimport type { LcmDag } from \"./dag.js\";\n\n/** Generate a ULID-like ID (timestamp + random). */\nfunction generateNodeId(): string {\n  const ts = Date.now().toString(36).padStart(9, \"0\");\n  const rand = Math.random().toString(36).slice(2, 10);\n  return `lcm-${ts}-${rand}`;\n}\n\nexport type SummarizeFn = (\n  text: string,\n  targetTokens: number,\n  aggressive: boolean,\n) => Promise<string | null>;\n\nexport interface LcmSummarizerConfig {\n  leafBatchSize: number;\n  rollupFanIn: number;\n  maxDepth: number;\n  deterministicMaxTokens: number;\n  telemetryPrefilterEnabled: boolean;\n}\n\nexport class LcmSummarizer {\n  constructor(\n    private readonly archive: LcmArchive,\n    private readonly dag: LcmDag,\n    private readonly summarizeFn: SummarizeFn,\n    private readonly config: LcmSummarizerConfig,\n  ) {}\n\n  /**\n   * Run incremental summarization for a session.\n   * Creates leaf nodes from unsummarized messages, then rolls up as needed.\n   */\n  async summarizeIncremental(sessionId: string): Promise<number> {\n    let nodesCreated = 0;\n\n    // Phase 1: Create leaf nodes from unsummarized messages\n    const unsummarized = this.archive.getUnsummarizedMessages(sessionId);\n    if (unsummarized.length >= this.config.leafBatchSize) {\n      const batches = chunkArray(unsummarized, this.config.leafBatchSize);\n      // Only process complete batches (leave partial for next time)\n      const completeBatches = unsummarized.length % this.config.leafBatchSize === 0\n        ? batches\n        : batches.slice(0, -1);\n\n      for (const batch of completeBatches) {\n        const created = await this.createLeafNode(sessionId, batch);\n        if (created) nodesCreated++;\n      }\n    }\n\n    // Phase 2: Roll up at each depth level\n    if (nodesCreated > 0) {\n      nodesCreated += await this.rollupAll(sessionId);\n    }\n\n    return nodesCreated;\n  }\n\n  /** Create a leaf summary node (depth 0) from a batch of messages. */\n  private async createLeafNode(\n    sessionId: string,\n    messages: LcmMessage[],\n  ): Promise<boolean> {\n    if (messages.length === 0) return false;\n\n    const combinedText = messages\n      .map((m) => `[${m.role}] ${m.content}`)\n      .join(\"\\n\\n\");\n\n    const inputTokens = estimateTokens(combinedText);\n    const targetTokens = Math.max(64, Math.ceil(inputTokens * 0.25));\n\n    const summary = await this.summarizeWithEscalation(combinedText, targetTokens);\n\n    this.dag.insertNode({\n      id: generateNodeId(),\n      session_id: sessionId,\n      depth: 0,\n      parent_id: null,\n      summary_text: summary.text,\n      token_count: estimateTokens(summary.text),\n      msg_start: messages[0].turn_index,\n      msg_end: messages[messages.length - 1].turn_index,\n      escalation: summary.escalation,\n    });\n\n    return true;\n  }\n\n  /** Roll up orphan nodes at each depth level up to maxDepth. */\n  private async rollupAll(sessionId: string): Promise<number> {\n    let totalCreated = 0;\n\n    for (let depth = 0; depth < this.config.maxDepth; depth++) {\n      const orphans = this.dag.getOrphanNodesAtDepth(sessionId, depth);\n      if (orphans.length < this.config.rollupFanIn) continue;\n\n      const batches = chunkArray(orphans, this.config.rollupFanIn);\n      // Only process complete batches\n      const completeBatches = orphans.length % this.config.rollupFanIn === 0\n        ? batches\n        : batches.slice(0, -1);\n\n      for (const batch of completeBatches) {\n        const combinedText = batch\n          .map((n) => n.summary_text)\n          .join(\"\\n\\n\");\n\n        const inputTokens = estimateTokens(combinedText);\n        const targetTokens = Math.max(64, Math.ceil(inputTokens * 0.25));\n\n        const summary = await this.summarizeWithEscalation(combinedText, targetTokens);\n\n        const parentId = generateNodeId();\n        this.dag.insertNode({\n          id: parentId,\n          session_id: sessionId,\n          depth: depth + 1,\n          parent_id: null,\n          summary_text: summary.text,\n          token_count: estimateTokens(summary.text),\n          msg_start: batch[0].msg_start,\n          msg_end: batch[batch.length - 1].msg_end,\n          escalation: summary.escalation,\n        });\n\n        this.dag.setParent(\n          batch.map((n) => n.id),\n          parentId,\n        );\n\n        totalCreated++;\n      }\n    }\n\n    return totalCreated;\n  }\n\n  /**\n   * Three-level escalation:\n   * 0 = Normal LLM summary\n   * 1 = Aggressive bullet compression\n   * 2 = Deterministic truncation (no LLM)\n   */\n  private async summarizeWithEscalation(\n    text: string,\n    targetTokens: number,\n  ): Promise<{ text: string; escalation: number }> {\n    const { telemetryPrefilterEnabled } = this.config;\n    if (\n      telemetryPrefilterEnabled &&\n      looksLikeMechanicalTelemetryTranscript(text)\n    ) {\n      return {\n        text: deterministicTruncate(\n          text,\n          Math.min(targetTokens, this.config.deterministicMaxTokens),\n        ),\n        escalation: 2,\n      };\n    }\n\n    // Level 0: Normal LLM summary\n    try {\n      const result = await this.summarizeFn(text, targetTokens, false);\n      if (result && estimateTokens(result) <= targetTokens * 1.5) {\n        return { text: result, escalation: 0 };\n      }\n      // If too long, try aggressive\n    } catch (err) {\n      log.debug(`LCM level-0 summary failed: ${err}`);\n    }\n\n    // Level 1: Aggressive bullet compression\n    try {\n      const aggressiveTarget = Math.max(32, Math.ceil(targetTokens * 0.5));\n      const result = await this.summarizeFn(text, aggressiveTarget, true);\n      if (result && estimateTokens(result) <= targetTokens * 1.5) {\n        return { text: result, escalation: 1 };\n      }\n    } catch (err) {\n      log.debug(`LCM level-1 summary failed: ${err}`);\n    }\n\n    // Level 2: Deterministic truncation (guaranteed, no LLM)\n    return {\n      text: deterministicTruncate(text, this.config.deterministicMaxTokens),\n      escalation: 2,\n    };\n  }\n}\n\n/** Deterministic truncation: first and last sentence, plus middle truncation. */\nfunction deterministicTruncate(text: string, maxTokens: number): string {\n  if (maxTokens <= 0 || estimateTokenCount(text) <= maxTokens) return text;\n\n  const sentences = text.split(/(?<=[.!?])\\s+/).filter((s) => s.length > 0);\n  if (sentences.length <= 2) return clampToTokenBudget(text, maxTokens);\n\n  const first = sentences[0];\n  const last = sentences[sentences.length - 1];\n  const render = (middle: string[], truncated: boolean): string =>\n    `${first} ${middle.join(\" \")}${truncated ? \" [...] \" : \" \"}${last}`;\n\n  const base = render([], false);\n  if (estimateTokenCount(base) > maxTokens) return clampToTokenBudget(text, maxTokens);\n  if (estimateTokenCount(render([], true)) > maxTokens) return base;\n\n  const middle: string[] = [];\n  for (let i = 1; i < sentences.length - 1; i++) {\n    const candidate = [...middle, sentences[i]];\n    const truncated = i < sentences.length - 2;\n    if (estimateTokenCount(render(candidate, truncated)) > maxTokens) break;\n    middle.push(sentences[i]);\n  }\n\n  const truncated = middle.length < sentences.length - 2;\n  return clampToTokenBudget(render(middle, truncated), maxTokens);\n}\n\nfunction clampToTokenBudget(text: string, maxTokens: number): string {\n  const codePoints = [...text];\n  let low = 0;\n  let high = codePoints.length;\n  while (low < high) {\n    const mid = Math.ceil((low + high) / 2);\n    if (estimateTokenCount(codePoints.slice(0, mid).join(\"\")) <= maxTokens) low = mid;\n    else high = mid - 1;\n  }\n  return codePoints.slice(0, low).join(\"\");\n}\n\nfunction chunkArray<T>(arr: T[], size: number): T[][] {\n  const chunks: T[][] = [];\n  for (let i = 0; i < arr.length; i += size) {\n    chunks.push(arr.slice(i, i + size));\n  }\n  return chunks;\n}\n"],"mappings":";;;;;;;;;;;;;;AAQA,SAAS,iBAAyB;AAChC,QAAM,KAAK,KAAK,IAAI,EAAE,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG;AAClD,QAAM,OAAO,KAAK,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,GAAG,EAAE;AACnD,SAAO,OAAO,EAAE,IAAI,IAAI;AAC1B;AAgBO,IAAM,gBAAN,MAAoB;AAAA,EACzB,YACmB,SACA,KACA,aACA,QACjB;AAJiB;AACA;AACA;AACA;AAAA,EAChB;AAAA,EAJgB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOnB,MAAM,qBAAqB,WAAoC;AAC7D,QAAI,eAAe;AAGnB,UAAM,eAAe,KAAK,QAAQ,wBAAwB,SAAS;AACnE,QAAI,aAAa,UAAU,KAAK,OAAO,eAAe;AACpD,YAAM,UAAU,WAAW,cAAc,KAAK,OAAO,aAAa;AAElE,YAAM,kBAAkB,aAAa,SAAS,KAAK,OAAO,kBAAkB,IACxE,UACA,QAAQ,MAAM,GAAG,EAAE;AAEvB,iBAAW,SAAS,iBAAiB;AACnC,cAAM,UAAU,MAAM,KAAK,eAAe,WAAW,KAAK;AAC1D,YAAI,QAAS;AAAA,MACf;AAAA,IACF;AAGA,QAAI,eAAe,GAAG;AACpB,sBAAgB,MAAM,KAAK,UAAU,SAAS;AAAA,IAChD;AAEA,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAc,eACZ,WACA,UACkB;AAClB,QAAI,SAAS,WAAW,EAAG,QAAO;AAElC,UAAM,eAAe,SAClB,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,KAAK,EAAE,OAAO,EAAE,EACrC,KAAK,MAAM;AAEd,UAAM,cAAc,eAAe,YAAY;AAC/C,UAAM,eAAe,KAAK,IAAI,IAAI,KAAK,KAAK,cAAc,IAAI,CAAC;AAE/D,UAAM,UAAU,MAAM,KAAK,wBAAwB,cAAc,YAAY;AAE7E,SAAK,IAAI,WAAW;AAAA,MAClB,IAAI,eAAe;AAAA,MACnB,YAAY;AAAA,MACZ,OAAO;AAAA,MACP,WAAW;AAAA,MACX,cAAc,QAAQ;AAAA,MACtB,aAAa,eAAe,QAAQ,IAAI;AAAA,MACxC,WAAW,SAAS,CAAC,EAAE;AAAA,MACvB,SAAS,SAAS,SAAS,SAAS,CAAC,EAAE;AAAA,MACvC,YAAY,QAAQ;AAAA,IACtB,CAAC;AAED,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAc,UAAU,WAAoC;AAC1D,QAAI,eAAe;AAEnB,aAAS,QAAQ,GAAG,QAAQ,KAAK,OAAO,UAAU,SAAS;AACzD,YAAM,UAAU,KAAK,IAAI,sBAAsB,WAAW,KAAK;AAC/D,UAAI,QAAQ,SAAS,KAAK,OAAO,YAAa;AAE9C,YAAM,UAAU,WAAW,SAAS,KAAK,OAAO,WAAW;AAE3D,YAAM,kBAAkB,QAAQ,SAAS,KAAK,OAAO,gBAAgB,IACjE,UACA,QAAQ,MAAM,GAAG,EAAE;AAEvB,iBAAW,SAAS,iBAAiB;AACnC,cAAM,eAAe,MAClB,IAAI,CAAC,MAAM,EAAE,YAAY,EACzB,KAAK,MAAM;AAEd,cAAM,cAAc,eAAe,YAAY;AAC/C,cAAM,eAAe,KAAK,IAAI,IAAI,KAAK,KAAK,cAAc,IAAI,CAAC;AAE/D,cAAM,UAAU,MAAM,KAAK,wBAAwB,cAAc,YAAY;AAE7E,cAAM,WAAW,eAAe;AAChC,aAAK,IAAI,WAAW;AAAA,UAClB,IAAI;AAAA,UACJ,YAAY;AAAA,UACZ,OAAO,QAAQ;AAAA,UACf,WAAW;AAAA,UACX,cAAc,QAAQ;AAAA,UACtB,aAAa,eAAe,QAAQ,IAAI;AAAA,UACxC,WAAW,MAAM,CAAC,EAAE;AAAA,UACpB,SAAS,MAAM,MAAM,SAAS,CAAC,EAAE;AAAA,UACjC,YAAY,QAAQ;AAAA,QACtB,CAAC;AAED,aAAK,IAAI;AAAA,UACP,MAAM,IAAI,CAAC,MAAM,EAAE,EAAE;AAAA,UACrB;AAAA,QACF;AAEA;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,wBACZ,MACA,cAC+C;AAC/C,UAAM,EAAE,0BAA0B,IAAI,KAAK;AAC3C,QACE,6BACA,uCAAuC,IAAI,GAC3C;AACA,aAAO;AAAA,QACL,MAAM;AAAA,UACJ;AAAA,UACA,KAAK,IAAI,cAAc,KAAK,OAAO,sBAAsB;AAAA,QAC3D;AAAA,QACA,YAAY;AAAA,MACd;AAAA,IACF;AAGA,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,YAAY,MAAM,cAAc,KAAK;AAC/D,UAAI,UAAU,eAAe,MAAM,KAAK,eAAe,KAAK;AAC1D,eAAO,EAAE,MAAM,QAAQ,YAAY,EAAE;AAAA,MACvC;AAAA,IAEF,SAAS,KAAK;AACZ,UAAI,MAAM,+BAA+B,GAAG,EAAE;AAAA,IAChD;AAGA,QAAI;AACF,YAAM,mBAAmB,KAAK,IAAI,IAAI,KAAK,KAAK,eAAe,GAAG,CAAC;AACnE,YAAM,SAAS,MAAM,KAAK,YAAY,MAAM,kBAAkB,IAAI;AAClE,UAAI,UAAU,eAAe,MAAM,KAAK,eAAe,KAAK;AAC1D,eAAO,EAAE,MAAM,QAAQ,YAAY,EAAE;AAAA,MACvC;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,MAAM,+BAA+B,GAAG,EAAE;AAAA,IAChD;AAGA,WAAO;AAAA,MACL,MAAM,sBAAsB,MAAM,KAAK,OAAO,sBAAsB;AAAA,MACpE,YAAY;AAAA,IACd;AAAA,EACF;AACF;AAGA,SAAS,sBAAsB,MAAc,WAA2B;AACtE,MAAI,aAAa,KAAK,mBAAmB,IAAI,KAAK,UAAW,QAAO;AAEpE,QAAM,YAAY,KAAK,MAAM,eAAe,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AACxE,MAAI,UAAU,UAAU,EAAG,QAAO,mBAAmB,MAAM,SAAS;AAEpE,QAAM,QAAQ,UAAU,CAAC;AACzB,QAAM,OAAO,UAAU,UAAU,SAAS,CAAC;AAC3C,QAAM,SAAS,CAACA,SAAkBC,eAChC,GAAG,KAAK,IAAID,QAAO,KAAK,GAAG,CAAC,GAAGC,aAAY,YAAY,GAAG,GAAG,IAAI;AAEnE,QAAM,OAAO,OAAO,CAAC,GAAG,KAAK;AAC7B,MAAI,mBAAmB,IAAI,IAAI,UAAW,QAAO,mBAAmB,MAAM,SAAS;AACnF,MAAI,mBAAmB,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,UAAW,QAAO;AAE7D,QAAM,SAAmB,CAAC;AAC1B,WAAS,IAAI,GAAG,IAAI,UAAU,SAAS,GAAG,KAAK;AAC7C,UAAM,YAAY,CAAC,GAAG,QAAQ,UAAU,CAAC,CAAC;AAC1C,UAAMA,aAAY,IAAI,UAAU,SAAS;AACzC,QAAI,mBAAmB,OAAO,WAAWA,UAAS,CAAC,IAAI,UAAW;AAClE,WAAO,KAAK,UAAU,CAAC,CAAC;AAAA,EAC1B;AAEA,QAAM,YAAY,OAAO,SAAS,UAAU,SAAS;AACrD,SAAO,mBAAmB,OAAO,QAAQ,SAAS,GAAG,SAAS;AAChE;AAEA,SAAS,mBAAmB,MAAc,WAA2B;AACnE,QAAM,aAAa,CAAC,GAAG,IAAI;AAC3B,MAAI,MAAM;AACV,MAAI,OAAO,WAAW;AACtB,SAAO,MAAM,MAAM;AACjB,UAAM,MAAM,KAAK,MAAM,MAAM,QAAQ,CAAC;AACtC,QAAI,mBAAmB,WAAW,MAAM,GAAG,GAAG,EAAE,KAAK,EAAE,CAAC,KAAK,UAAW,OAAM;AAAA,QACzE,QAAO,MAAM;AAAA,EACpB;AACA,SAAO,WAAW,MAAM,GAAG,GAAG,EAAE,KAAK,EAAE;AACzC;AAEA,SAAS,WAAc,KAAU,MAAqB;AACpD,QAAM,SAAgB,CAAC;AACvB,WAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK,MAAM;AACzC,WAAO,KAAK,IAAI,MAAM,GAAG,IAAI,IAAI,CAAC;AAAA,EACpC;AACA,SAAO;AACT;","names":["middle","truncated"]}