@dxos/assistant 0.8.4-main.fd6878d → 0.8.4-staging.60fe92afc8

package/src/util/execution-graph.ts

@@ -0,0 +1,520 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { AgentStatus } from '@dxos/ai';
+import { type Entity, Obj, type Ref } from '@dxos/echo';
+import { MESSAGE_PROPERTY_TOOL_CALL_ID } from '@dxos/functions';
+import { EID, type EntityId } from '@dxos/keys';
+import { LogLevel } from '@dxos/log';
+import { ContentBlock, Message } from '@dxos/types';
+import { isNonNullable } from '@dxos/util';
+
+const SKIP_BLOCKS: ContentBlock.Any['_tag'][] = ['text'];
+
+/**
+ * Mercurial-style Commit.
+ */
+// TODO(wittjosiah): Reconcile with @dxos/react-ui-components.
+export type Commit = {
+  id: string;
+  parents: string[];
+  branch: string;
+  icon?: string;
+  level?: LogLevel;
+  message: string;
+  timestamp?: Date;
+  tags?: string[];
+};
+
+// TODO(burdon): Add colors?
+enum IconType {
+  // General status.
+  SUCCESS = 'ph--check-circle--regular',
+  WARNING = 'ph--warning-circle--regular',
+  ERROR = 'ph--x-circle--regular',
+  FLAG = 'ph--flag--regular',
+  ROCKET = 'ph--rocket--regular',
+  TIMER = 'ph--timer--regular',
+
+  // Interactions.
+  USER = 'ph--user--regular',
+  USER_INTERACTION = 'ph--user-sound--regular',
+  AGENT = 'ph--drone--regular',
+  THINKING = 'ph--brain--regular',
+  LINK = 'ph--link--regular',
+  TOOL = 'ph--wrench--regular',
+}
+
+/**
+ * Models the execution graph based on a stream of events.
+ */
+export class ExecutionGraph {
+  // TODO(dmaretskyi): Evolve the internal state to support chats, functions, circuit workflows, etc.
+  private _commits: Commit[] = [];
+  private _branchNames = new Set<string>();
+  private _lastBlockId: string | undefined;
+
+  // Tool call tracking.
+  private _toolCallCommitIds = new Map<string, string>(); // toolCallId -> commitId
+  private _lastCommitByBranch = new Map<string, string>(); // branch -> last commitId
+  private _pendingToolResults = new Map<string, string>(); // toolCallId -> toolResultCommitId
+
+  constructor(private readonly _skipBlocks: ContentBlock.Any['_tag'][] = SKIP_BLOCKS) {}
+
+  /**
+   * Adds events to the graph.
+   */
+  addEvents(events: Entity.Unknown[]) {
+    const sortedEvents = this.sortObjectsByCreated(events);
+    for (const event of sortedEvents) {
+      if (Obj.instanceOf(Message.Message, event)) {
+        this._processMessage(event);
+      } else if (Obj.instanceOf(AgentStatus, event)) {
+        this._processAgentStatus(event);
+      }
+    }
+  }
+
+  /**
+   * Processes a message event and creates commits for its blocks.
+   */
+  private _processMessage(message: Message.Message) {
+    const messageCommits = messageToCommits(
+      message,
+      this._lastBlockId,
+      this._toolCallCommitIds,
+      this._lastCommitByBranch,
+      this._skipBlocks,
+    );
+    this._commits.push(...messageCommits);
+
+    // Track branches.
+    messageCommits.forEach((commit) => this._branchNames.add(commit.branch));
+
+    // Track tool calls and update pending tool results.
+    this._trackToolCalls(messageCommits);
+
+    // Update last block ID for sequential chaining.
+    if (messageCommits.length > 0) {
+      this._lastBlockId = messageCommits[messageCommits.length - 1].id;
+    }
+  }
+
+  /**
+   * Processes an AgentStatus event.
+   */
+  private _processAgentStatus(event: AgentStatus) {
+    const branch = this._getToolCallBranch(event.parentMessage, event.toolCallId);
+    this._branchNames.add(branch);
+
+    const agentStatusCommit: Commit = {
+      id: event.id,
+      branch,
+      message: event.message,
+      icon: IconType.FLAG,
+      parents: this._getAgentStatusParents(event),
+      timestamp: new Date(event.created),
+    };
+
+    this._commits.push(agentStatusCommit);
+
+    // Update last block ID for sequential chaining.
+    this._lastBlockId = event.id;
+
+    // Update pending tool results to point to this AgentStatus.
+    if (event.toolCallId) {
+      this._lastCommitByBranch.set(branch, event.id);
+      this._updatePendingToolResults(event.toolCallId, event.id);
+    }
+  }
+
+  /**
+   * Stable sort: objects with 'created' field are sorted by it, others remain in place.
+   */
+  private sortObjectsByCreated(objects: any[]) {
+    const sortedObjects = objects.slice();
+    // Find indices of objects with 'created'
+    const createdObjects: { idx: number; obj: any }[] = [];
+    sortedObjects.forEach((obj, idx) => {
+      if (obj && typeof obj.created === 'string') {
+        createdObjects.push({ idx, obj });
+      }
+    });
+    // Sort only the objects with 'created' by their date, stable
+    createdObjects.sort((a, b) => {
+      const aDate = new Date(a.obj.created).getTime();
+      const bDate = new Date(b.obj.created).getTime();
+      return aDate - bDate;
+    });
+    // Place sorted createdObjects back into their original positions
+    let createdIdx = 0;
+    return sortedObjects.map((obj, _idx) => {
+      if (obj && typeof obj.created === 'string') {
+        return createdObjects[createdIdx++].obj;
+      }
+      return obj;
+    });
+  }
+
+  /**
+   * Returns the current state of the graph.
+   */
+  getGraph(lastRequest = false): { branches: string[]; commits: Commit[] } {
+    const idx = lastRequest ? this._commits.findLastIndex((c) => c.tags?.includes('user')) : -1;
+    const commits = idx === -1 ? this._commits : this._commits.slice(idx);
+
+    return {
+      branches: Array.from(this._branchNames),
+      commits,
+    };
+  }
+
+  /**
+   * Tracks tool calls and handles pending tool result updates.
+   */
+  private _trackToolCalls(commits: Commit[]) {
+    commits.forEach((commit) => {
+      if (this._isToolCallCommit(commit)) {
+        const toolCallId = this._extractToolCallId(commit.id);
+        if (toolCallId) {
+          this._toolCallCommitIds.set(toolCallId, commit.id);
+        }
+      }
+
+      if (this._isToolResultCommit(commit)) {
+        const toolCallId = this._extractToolCallId(commit.id);
+        if (toolCallId) {
+          this._pendingToolResults.set(toolCallId, commit.id);
+          this._handleLateToolResultUpdate(toolCallId, commit.id);
+        }
+      }
+
+      this._lastCommitByBranch.set(commit.branch, commit.id);
+    });
+  }
+
+  /**
+   * Updates pending tool results to point to an AgentStatus event.
+   */
+  private _updatePendingToolResults(toolCallId: string, agentStatusId: string) {
+    const toolResultId = this._pendingToolResults.get(toolCallId);
+    if (!toolResultId) {
+      return;
+    }
+
+    const toolResultCommit = this._commits.find((c) => c.id === toolResultId);
+    if (!toolResultCommit) {
+      return;
+    }
+
+    const toolCallCommitId = this._toolCallCommitIds.get(toolCallId);
+    if (!toolCallCommitId) {
+      return;
+    }
+
+    // Replace tool call commit with AgentStatus commit.
+    const toolCallIndex = toolResultCommit.parents.indexOf(toolCallCommitId);
+    if (toolCallIndex !== -1) {
+      toolResultCommit.parents[toolCallIndex] = agentStatusId;
+    }
+
+    this._pendingToolResults.delete(toolCallId);
+  }
+
+  /**
+   * Handles late updates when AgentStatus is processed before tool result.
+   */
+  private _handleLateToolResultUpdate(toolCallId: string, toolResultId: string) {
+    const toolCallCommitId = this._toolCallCommitIds.get(toolCallId);
+    if (!toolCallCommitId) {
+      return;
+    }
+
+    const messageId = toolCallCommitId.split('_toolCall_')[0];
+    const toolCallBranch = `${messageId}_${toolCallId}`;
+    const agentStatusCommitId = this._lastCommitByBranch.get(toolCallBranch);
+
+    if (agentStatusCommitId && agentStatusCommitId !== toolCallCommitId) {
+      const toolResultCommit = this._commits.find((c) => c.id === toolResultId);
+      if (toolResultCommit) {
+        const toolCallIndex = toolResultCommit.parents.indexOf(toolCallCommitId);
+        if (toolCallIndex !== -1) {
+          toolResultCommit.parents[toolCallIndex] = agentStatusCommitId;
+        }
+      }
+      this._pendingToolResults.delete(toolCallId);
+    }
+  }
+
+  /**
+   * Gets the branch name for a tool call.
+   */
+  private _getToolCallBranch(parentMessage?: EntityId, toolCallId?: string): string {
+    if (toolCallId) {
+      return toolCallId;
+    }
+    return parentMessage || 'main';
+  }
+
+  /**
+   * Gets the parents for an AgentStatus commit.
+   */
+  private _getAgentStatusParents(event: AgentStatus): string[] {
+    if (event.parentMessage && event.toolCallId) {
+      return [getToolCallId(event.parentMessage, event.toolCallId)];
+    }
+    return [];
+  }
+
+  /**
+   * Checks if a commit is a tool call commit.
+   */
+  private _isToolCallCommit(commit: Commit): boolean {
+    return commit.message.includes('Calling tool');
+  }
+
+  /**
+   * Checks if a commit is a tool result commit.
+   */
+  private _isToolResultCommit(commit: Commit): boolean {
+    return commit.message.includes('Tool call succeeded') || commit.message.includes('Tool error:');
+  }
+
+  /**
+   * Extracts tool call ID from a commit ID.
+   */
+  private _extractToolCallId(commitId: string): string | undefined {
+    const toolCallMatch = commitId.match(/_toolCall_(.+)$/);
+    const toolResultMatch = commitId.match(/_toolResult_(.+)$/);
+    return toolCallMatch?.[1] || toolResultMatch?.[1];
+  }
+}
+
+/**
+ * Creates commits for all blocks in a message.
+ */
+// TODO(burdon): Pass in AiToolProvider.
+const messageToCommits = (
+  message: Message.Message,
+  lastBlockId?: string,
+  toolCallIds?: Map<string, string>,
+  lastCommitByBranch?: Map<string, string>,
+  skipBlocks?: ContentBlock.Any['_tag'][],
+): Commit[] => {
+  let previousBlockId: string | undefined = lastBlockId;
+
+  return message.blocks
+    .map((block, idx) => {
+      const branch = getMessageBranch(message);
+      const parents = getBlockParents(block, previousBlockId, message, toolCallIds, lastCommitByBranch);
+      if (skipBlocks?.includes(block._tag)) {
+        return null;
+      }
+
+      const commit = createBlockCommit(block, message, branch, parents, idx);
+      if (commit) {
+        previousBlockId = commit.id;
+      }
+
+      return commit;
+    })
+    .filter(isNonNullable);
+};
+
+/**
+ * Determines the parent commits for a block based on its type and context.
+ */
+const getBlockParents = (
+  block: ContentBlock.Any,
+  previousBlockId: string | undefined,
+  message: Message.Message,
+  toolCallIds?: Map<string, string>,
+  lastCommitByBranch?: Map<string, string>,
+): string[] => {
+  const parents: string[] = [];
+
+  if (block._tag === 'toolResult') {
+    // Tool results have two parents: previous block and last block from tool call branch.
+    if (previousBlockId) {
+      parents.push(previousBlockId);
+    }
+
+    const toolCallCommitId = toolCallIds?.get(block.toolCallId);
+    if (toolCallCommitId) {
+      const toolCallBranch = getToolCallBranchFromCommitId(toolCallCommitId, block.toolCallId);
+      const lastBlockFromToolCallBranch = lastCommitByBranch?.get(toolCallBranch);
+
+      if (lastBlockFromToolCallBranch) {
+        parents.push(lastBlockFromToolCallBranch);
+      } else {
+        // Fallback to tool call commit if no AgentStatus exists yet.
+        parents.push(toolCallCommitId);
+      }
+    }
+  } else {
+    // For other blocks, use previous block as parent (or tool call parent for first block).
+    if (previousBlockId) {
+      parents.push(previousBlockId);
+    } else {
+      const toolCallParent = getParentId(message);
+      if (toolCallParent) {
+        parents.push(toolCallParent);
+      }
+    }
+  }
+
+  return parents;
+};
+
+/**
+ * Creates a commit for a specific block.
+ */
+const createBlockCommit = (
+  block: ContentBlock.Any,
+  message: Message.Message,
+  branch: string,
+  parents: string[],
+  idx: number,
+): Commit | null => {
+  const timestamp = new Date(message.created);
+  switch (block._tag) {
+    case 'text':
+      if (!block.text.trim().length) {
+        return null;
+      }
+      return {
+        id: getGenericBlockId(message.id, idx),
+        branch,
+        parents,
+        timestamp,
+        ...(message.sender.role === 'user'
+          ? {
+              icon: IconType.USER,
+              tags: ['user'],
+              message: 'Processing request...',
+            }
+          : {
+              icon: IconType.AGENT,
+              message: `Response (${block.text.split(' ').length} words)`,
+            }),
+      } satisfies Commit;
+
+    case 'toolCall':
+      return {
+        id: getToolCallId(message.id, block.toolCallId),
+        branch,
+        parents,
+        timestamp,
+        icon: IconType.TOOL,
+        level: LogLevel.INFO,
+        message: `Calling tool (${block.name})`,
+      } satisfies Commit;
+
+    case 'toolResult':
+      return {
+        id: getToolResultId(message.id, block.toolCallId),
+        branch,
+        parents,
+        timestamp,
+        icon: block.error ? IconType.ERROR : IconType.SUCCESS,
+        level: block.error ? LogLevel.ERROR : LogLevel.INFO,
+        message: block.error ? 'Tool error: ' + block.error : 'Tool call succeeded',
+      } satisfies Commit;
+
+    case 'stats':
+      return {
+        id: getGenericBlockId(message.id, idx),
+        branch,
+        parents,
+        timestamp,
+        icon: IconType.ROCKET,
+        level: LogLevel.INFO,
+        message: ContentBlock.createStatsMessage(block),
+      } satisfies Commit;
+
+    case 'status':
+      return {
+        id: getGenericBlockId(message.id, idx),
+        branch,
+        parents,
+        timestamp,
+        message: block.statusText,
+        level: LogLevel.INFO,
+        icon: IconType.FLAG,
+      } satisfies Commit;
+
+    case 'reasoning':
+      return {
+        id: getGenericBlockId(message.id, idx),
+        branch,
+        parents,
+        timestamp,
+        message: block.reasoningText ?? 'Thinking...',
+        icon: IconType.THINKING,
+      } satisfies Commit;
+
+    case 'reference':
+      return {
+        id: getGenericBlockId(message.id, idx),
+        branch,
+        parents,
+        timestamp,
+        icon: IconType.LINK,
+        message: stringifyRef(block.reference),
+      } satisfies Commit;
+
+    default:
+      return null;
+  }
+};
+
+/**
+ * Gets the tool call branch name from a tool call commit ID.
+ */
+const getToolCallBranchFromCommitId = (toolCallCommitId: string, toolCallId: string): string => {
+  const messageId = toolCallCommitId.split('_toolCall_')[0];
+  return `${messageId}_${toolCallId}`;
+};
+
+const getToolCallId = (messageId: EntityId, toolCallId: string) => `${messageId}_toolCall_${toolCallId}`;
+const getToolResultId = (messageId: EntityId, toolCallId: string) => `${messageId}_toolResult_${toolCallId}`;
+const getGenericBlockId = (messageId: EntityId, idx: number) => `${messageId}_block_${idx}`;
+
+const getBranchName = (options: { parentMessage?: EntityId; toolCallId?: string }) => {
+  if (options.toolCallId) {
+    return options.toolCallId;
+  } else if (options.parentMessage) {
+    return options.parentMessage;
+  } else {
+    return 'main';
+  }
+};
+
+const getMessageBranch = (message: Message.Message) => {
+  return getBranchName({
+    parentMessage: message.parentMessage,
+    toolCallId: message.properties?.[MESSAGE_PROPERTY_TOOL_CALL_ID],
+  });
+};
+
+const getParentId = (message: Message.Message) => {
+  if (message.parentMessage && message.properties?.[MESSAGE_PROPERTY_TOOL_CALL_ID]) {
+    return getToolCallId(message.parentMessage, message.properties[MESSAGE_PROPERTY_TOOL_CALL_ID]);
+  } else {
+    return undefined;
+  }
+};
+
+const stringifyRef = (ref: Ref.Unknown) => {
+  if (ref.target) {
+    return stringifyObject(ref.target);
+  }
+
+  const echoUri = EID.tryParse(ref.uri);
+  return (echoUri ? EID.getEntityId(echoUri) : undefined) ?? '';
+};
+
+const stringifyObject = (obj: Obj.Unknown) => {
+  return Obj.getLabel(obj) ?? Obj.getTypename(obj) ?? obj.id;
+};

package/src/util/index.ts

@@ -0,0 +1,10 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './artifact';
+export * from './diff';
+export * from './error-handling';
+export * from './errors';
+export * from './execution-graph';
+export * from './tracing';

package/src/util/tracing.ts

@@ -0,0 +1,60 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { Trace } from '@dxos/compute';
+import { Obj } from '@dxos/echo';
+import { ContentBlock, Actor } from '@dxos/types';
+
+/**
+ * Partial content block emitted.
+ */
+export const PartialBlock = Trace.EventType('assistant.partialBlock', {
+  schema: Schema.Struct({
+    messageId: Obj.ID,
+    role: Actor.Role,
+    block: ContentBlock.Any,
+  }),
+  isEphemeral: true,
+});
+
+/**
+ * Complete content block emitted.
+ */
+export const CompleteBlock = Trace.EventType('assistant.completeBlock', {
+  schema: Schema.Struct({
+    messageId: Obj.ID,
+    role: Actor.Role,
+    block: ContentBlock.Any,
+  }),
+  isEphemeral: false,
+});
+
+export const AgentRequestBegin = Trace.EventType('assistant.agentRequestBegin', {
+  schema: Schema.Struct({}),
+  isEphemeral: false,
+});
+
+export const AgentRequestEnd = Trace.EventType('assistant.agentRequestEnd', {
+  schema: Schema.Struct({
+    status: Schema.Literal('success', 'error', 'interrupted'),
+    error: Schema.optional(Schema.String),
+  }),
+  isEphemeral: false,
+});
+
+/**
+ * Emitted when an MCP server connection fails for a request turn.
+ * Ephemeral so that misconfigured/unreachable servers don't pollute the durable feed,
+ * but can still be surfaced to the user via the live ephemeral event stream.
+ */
+export const McpServerError = Trace.EventType('assistant.mcpServerError', {
+  schema: Schema.Struct({
+    url: Schema.String,
+    protocol: Schema.Literal('sse', 'http'),
+    message: Schema.String,
+  }),
+  isEphemeral: true,
+});

package/dist/lib/browser/chunk-6WRGDEMH.mjs

@@ -1,89 +0,0 @@
-// src/extraction/extraction-ner-function/named-entity-recognition.ts
-import { pipeline } from "@xenova/transformers";
-import { log } from "@dxos/log";
-var __dxlog_file = "/__w/dxos/dxos/packages/core/assistant/src/extraction/extraction-ner-function/named-entity-recognition.ts";
-var _ner;
-var getNer = () => {
-  if (!_ner) {
-    _ner = pipeline("ner", "Xenova/bert-base-NER").then((ner) => {
-      log.info("NER model is ready", void 0, {
-        F: __dxlog_file,
-        L: 23,
-        S: void 0,
-        C: (f, a) => f(...a)
-      });
-      return ner;
-    });
-  }
-  return _ner;
-};
-var extractFullEntities = async (text) => {
-  const ner = await getNer();
-  const entities = await ner(text);
-  const tokenGroups = createTokenGroups(entities);
-  const combinedEntities = tokenGroups.map(combineNerTokens);
-  return combinedEntities;
-};
-var createTokenGroups = (tokens) => {
-  const tokenGroups = [];
-  const canBeCombined = (prevToken, currentToken) => {
-    const [currentState, currentEntity] = currentToken.entity.split("-");
-    const [prevState, prevEntity] = prevToken.entity.split("-");
-    if (prevEntity !== currentEntity) {
-      return false;
-    }
-    if (prevState === "O") {
-      return false;
-    }
-    if (currentState === "I") {
-      return true;
-    }
-    if ((prevState === "B" || prevState === "I") && currentState === "B") {
-      return currentToken.word.startsWith("#");
-    }
-    return false;
-  };
-  for (const token of tokens) {
-    if (tokenGroups.length === 0) {
-      tokenGroups.push([
-        token
-      ]);
-      continue;
-    }
-    const lastTokenGroup = tokenGroups.at(-1);
-    if (canBeCombined(lastTokenGroup.at(-1), token)) {
-      lastTokenGroup.push(token);
-      continue;
-    }
-    tokenGroups.push([
-      token
-    ]);
-  }
-  return tokenGroups;
-};
-var combineNerTokens = (group) => {
-  const combinedToken = {
-    entity: group.at(0).entity.split("-")[1],
-    score: group.reduce((acc, token) => acc + token.score, 0) / group.length,
-    index: group.at(0).index,
-    word: group.at(0).word,
-    start: group.at(0).start,
-    end: group.at(-1).end
-  };
-  for (const token of group.slice(1)) {
-    if (token.word.startsWith("#")) {
-      combinedToken.word += token.word.replace(/^#+/, "");
-    } else {
-      combinedToken.word += " " + token.word;
-    }
-  }
-  return combinedToken;
-};
-
-export {
-  getNer,
-  extractFullEntities,
-  createTokenGroups,
-  combineNerTokens
-};
-//# sourceMappingURL=chunk-6WRGDEMH.mjs.map

package/dist/lib/browser/chunk-6WRGDEMH.mjs.map

@@ -1,7 +0,0 @@
-{
-  "version": 3,
-  "sources": ["../../../src/extraction/extraction-ner-function/named-entity-recognition.ts"],
-  "sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport {\n  type TokenClassificationOutput,\n  type TokenClassificationPipelineType,\n  type TokenClassificationSingle,\n  pipeline,\n} from '@xenova/transformers';\n\nimport { log } from '@dxos/log';\n\nlet _ner: Promise<TokenClassificationPipelineType>;\n/**\n * Named Entity Recognition pipeline.\n * Initializes the pipeline on first call.\n * @returns The singleton promise that resolves to a token classification pipeline.\n */\nexport const getNer = () => {\n  if (!_ner) {\n    _ner = pipeline('ner', 'Xenova/bert-base-NER').then((ner) => {\n      log.info('NER model is ready');\n      return ner;\n    });\n  }\n  return _ner;\n};\n\n/**\n * Extracts entities from text using the named entity recognition pipeline and does simple grouping of tokens.\n */\nexport const extractFullEntities = async (text: string): Promise<TokenClassificationSingle[]> => {\n  const ner = await getNer();\n  const entities = (await ner(text)) as TokenClassificationOutput;\n  const tokenGroups = createTokenGroups(entities);\n  const combinedEntities = tokenGroups.map(combineNerTokens);\n  return combinedEntities;\n};\n\nexport const createTokenGroups = (tokens: TokenClassificationSingle[]): TokenClassificationSingle[][] => {\n  const tokenGroups: TokenClassificationSingle[][] = [];\n\n  const canBeCombined = (prevToken: TokenClassificationSingle, currentToken: TokenClassificationSingle) => {\n    const [currentState, currentEntity] = currentToken.entity.split('-'); // B, I, O\n    const [prevState, prevEntity] = prevToken.entity.split('-'); // B, I, O\n    if (prevEntity !== currentEntity) {\n      return false;\n    }\n    if (prevState === 'O') {\n      return false;\n    }\n    if (currentState === 'I') {\n      return true;\n    }\n    if ((prevState === 'B' || prevState === 'I') && currentState === 'B') {\n      return currentToken.word.startsWith('#');\n    }\n    return false;\n  };\n\n  for (const token of tokens) {\n    if (tokenGroups.length === 0) {\n      tokenGroups.push([token]);\n      continue;\n    }\n    const lastTokenGroup = tokenGroups.at(-1)!;\n    if (canBeCombined(lastTokenGroup.at(-1)!, token)) {\n      lastTokenGroup.push(token);\n      continue;\n    }\n    tokenGroups.push([token]);\n  }\n\n  return tokenGroups;\n};\n\nexport const combineNerTokens = (group: TokenClassificationSingle[]): TokenClassificationSingle => {\n  const combinedToken: TokenClassificationSingle = {\n    entity: group.at(0)!.entity.split('-')[1],\n    score: group.reduce((acc, token) => acc + token.score, 0) / group.length,\n    index: group.at(0)!.index,\n    word: group.at(0)!.word,\n    start: group.at(0)!.start,\n    end: group.at(-1)!.end,\n  };\n\n  for (const token of group.slice(1)) {\n    if (token.word.startsWith('#')) {\n      combinedToken.word += token.word.replace(/^#+/, '');\n    } else {\n      combinedToken.word += ' ' + token.word;\n    }\n  }\n\n  return combinedToken;\n};\n"],
-  "mappings": ";AAIA,SAIEA,gBACK;AAEP,SAASC,WAAW;;AAEpB,IAAIC;AAMG,IAAMC,SAAS,MAAA;AACpB,MAAI,CAACD,MAAM;AACTA,WAAOF,SAAS,OAAO,sBAAA,EAAwBI,KAAK,CAACC,QAAAA;AACnDJ,UAAIK,KAAK,sBAAA,QAAA;;;;;;AACT,aAAOD;IACT,CAAA;EACF;AACA,SAAOH;AACT;AAKO,IAAMK,sBAAsB,OAAOC,SAAAA;AACxC,QAAMH,MAAM,MAAMF,OAAAA;AAClB,QAAMM,WAAY,MAAMJ,IAAIG,IAAAA;AAC5B,QAAME,cAAcC,kBAAkBF,QAAAA;AACtC,QAAMG,mBAAmBF,YAAYG,IAAIC,gBAAAA;AACzC,SAAOF;AACT;AAEO,IAAMD,oBAAoB,CAACI,WAAAA;AAChC,QAAML,cAA6C,CAAA;AAEnD,QAAMM,gBAAgB,CAACC,WAAsCC,iBAAAA;AAC3D,UAAM,CAACC,cAAcC,aAAAA,IAAiBF,aAAaG,OAAOC,MAAM,GAAA;AAChE,UAAM,CAACC,WAAWC,UAAAA,IAAcP,UAAUI,OAAOC,MAAM,GAAA;AACvD,QAAIE,eAAeJ,eAAe;AAChC,aAAO;IACT;AACA,QAAIG,cAAc,KAAK;AACrB,aAAO;IACT;AACA,QAAIJ,iBAAiB,KAAK;AACxB,aAAO;IACT;AACA,SAAKI,cAAc,OAAOA,cAAc,QAAQJ,iBAAiB,KAAK;AACpE,aAAOD,aAAaO,KAAKC,WAAW,GAAA;IACtC;AACA,WAAO;EACT;AAEA,aAAWC,SAASZ,QAAQ;AAC1B,QAAIL,YAAYkB,WAAW,GAAG;AAC5BlB,kBAAYmB,KAAK;QAACF;OAAM;AACxB;IACF;AACA,UAAMG,iBAAiBpB,YAAYqB,GAAG,EAAC;AACvC,QAAIf,cAAcc,eAAeC,GAAG,EAAC,GAAKJ,KAAAA,GAAQ;AAChDG,qBAAeD,KAAKF,KAAAA;AACpB;IACF;AACAjB,gBAAYmB,KAAK;MAACF;KAAM;EAC1B;AAEA,SAAOjB;AACT;AAEO,IAAMI,mBAAmB,CAACkB,UAAAA;AAC/B,QAAMC,gBAA2C;IAC/CZ,QAAQW,MAAMD,GAAG,CAAA,EAAIV,OAAOC,MAAM,GAAA,EAAK,CAAA;IACvCY,OAAOF,MAAMG,OAAO,CAACC,KAAKT,UAAUS,MAAMT,MAAMO,OAAO,CAAA,IAAKF,MAAMJ;IAClES,OAAOL,MAAMD,GAAG,CAAA,EAAIM;IACpBZ,MAAMO,MAAMD,GAAG,CAAA,EAAIN;IACnBa,OAAON,MAAMD,GAAG,CAAA,EAAIO;IACpBC,KAAKP,MAAMD,GAAG,EAAC,EAAIQ;EACrB;AAEA,aAAWZ,SAASK,MAAMQ,MAAM,CAAA,GAAI;AAClC,QAAIb,MAAMF,KAAKC,WAAW,GAAA,GAAM;AAC9BO,oBAAcR,QAAQE,MAAMF,KAAKgB,QAAQ,OAAO,EAAA;IAClD,OAAO;AACLR,oBAAcR,QAAQ,MAAME,MAAMF;IACpC;EACF;AAEA,SAAOQ;AACT;",
-  "names": ["pipeline", "log", "_ner", "getNer", "then", "ner", "info", "extractFullEntities", "text", "entities", "tokenGroups", "createTokenGroups", "combinedEntities", "map", "combineNerTokens", "tokens", "canBeCombined", "prevToken", "currentToken", "currentState", "currentEntity", "entity", "split", "prevState", "prevEntity", "word", "startsWith", "token", "length", "push", "lastTokenGroup", "at", "group", "combinedToken", "score", "reduce", "acc", "index", "start", "end", "slice", "replace"]
-}