@agentmark-ai/shared-utils 0.6.1 → 0.7.0

package/dist/index.mjs

@@ -2136,6 +2136,520 @@ var OtelGenAiTransformer = class {
 };
 OtelGenAiTransformer.SCOPE_NAME = "pydantic-ai";
 
+// src/normalizer/extractors/indexed-message-parser.ts
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function collectIndices(attributes, prefix) {
+  const re = new RegExp(`^${escapeRegExp(prefix)}\\.(\\d+)\\.`);
+  const seen = /* @__PURE__ */ new Set();
+  for (const key of Object.keys(attributes)) {
+    const match = re.exec(key);
+    if (match) seen.add(Number(match[1]));
+  }
+  return [...seen].sort((a, b) => a - b);
+}
+function parseArgs(raw) {
+  if (raw === void 0 || raw === null) return {};
+  if (typeof raw === "object") return raw;
+  if (typeof raw === "string") {
+    try {
+      const parsed = JSON.parse(raw);
+      return parsed && typeof parsed === "object" ? parsed : { raw };
+    } catch {
+      return { raw };
+    }
+  }
+  return { raw };
+}
+function readMessageToolCalls(attributes, messageBase, config) {
+  const tc = config.toolCalls;
+  if (!tc) return [];
+  const arrayPrefix = `${messageBase}${tc.arrayKey}`;
+  const indices = collectIndices(attributes, arrayPrefix);
+  const calls = [];
+  for (const j of indices) {
+    const base = `${arrayPrefix}.${j}.${tc.infix}`;
+    const name = attributes[`${base}${tc.nameKey}`];
+    if (name === void 0) continue;
+    const id = attributes[`${base}${tc.idKey}`];
+    const args = parseArgs(attributes[`${base}${tc.argsKey}`]);
+    calls.push({
+      type: "tool-call",
+      toolCallId: id !== void 0 ? String(id) : "",
+      toolName: String(name),
+      args
+    });
+  }
+  return calls;
+}
+function readContentsText(attributes, messageBase, config) {
+  var _a;
+  if (!config.contentsKey) return void 0;
+  const contentsPrefix = `${messageBase}${config.contentsKey}`;
+  const indices = collectIndices(attributes, contentsPrefix);
+  if (indices.length === 0) return void 0;
+  const partInfix = (_a = config.contentsPartInfix) != null ? _a : "";
+  const parts = [];
+  for (const j of indices) {
+    const text = attributes[`${contentsPrefix}.${j}.${partInfix}text`];
+    if (typeof text === "string" && text.length > 0) parts.push(text);
+  }
+  return parts.length > 0 ? parts.join("\n") : void 0;
+}
+function parseIndexedMessages(attributes, config) {
+  var _a, _b;
+  const indices = collectIndices(attributes, config.prefix);
+  if (indices.length === 0) return void 0;
+  const roleKey = (_a = config.roleKey) != null ? _a : "role";
+  const contentKey = (_b = config.contentKey) != null ? _b : "content";
+  const messages = [];
+  for (const i of indices) {
+    const base = `${config.prefix}.${i}.${config.messageInfix}`;
+    const role = attributes[`${base}${roleKey}`];
+    const scalarContent = attributes[`${base}${contentKey}`];
+    const text = typeof scalarContent === "string" ? scalarContent : scalarContent !== void 0 ? String(scalarContent) : readContentsText(attributes, base, config);
+    const toolCalls = readMessageToolCalls(attributes, base, config);
+    if (role === void 0 && text === void 0 && toolCalls.length === 0) continue;
+    let content;
+    if (toolCalls.length > 0) {
+      const parts = [];
+      if (text) parts.push({ type: "text", text });
+      for (const call of toolCalls) {
+        parts.push({
+          type: "tool-call",
+          toolCallId: call.toolCallId,
+          toolName: call.toolName,
+          args: call.args
+        });
+      }
+      content = parts;
+    } else {
+      content = text != null ? text : "";
+    }
+    messages.push({ role: role !== void 0 ? String(role) : "user", content });
+  }
+  return messages.length > 0 ? messages : void 0;
+}
+function extractIndexedToolCalls(attributes, config) {
+  if (!config.toolCalls) return void 0;
+  const indices = collectIndices(attributes, config.prefix);
+  const calls = [];
+  for (const i of indices) {
+    const base = `${config.prefix}.${i}.${config.messageInfix}`;
+    calls.push(...readMessageToolCalls(attributes, base, config));
+  }
+  return calls.length > 0 ? calls : void 0;
+}
+function messagesToPlainText(messages) {
+  if (!messages || messages.length === 0) return void 0;
+  const chunks = [];
+  for (const msg of messages) {
+    if (typeof msg.content === "string") {
+      if (msg.content.length > 0) chunks.push(msg.content);
+    } else if (Array.isArray(msg.content)) {
+      for (const part of msg.content) {
+        if (typeof part === "string") {
+          if (part.length > 0) chunks.push(part);
+        } else if (part.type === "text" && part.text.length > 0) {
+          chunks.push(part.text);
+        }
+      }
+    }
+  }
+  return chunks.length > 0 ? chunks.join("\n") : void 0;
+}
+
+// src/normalizer/utils/coerce.ts
+function toNumber(value) {
+  if (typeof value === "number") return Number.isFinite(value) ? value : void 0;
+  if (typeof value === "string") {
+    const n = Number(value);
+    return Number.isFinite(n) ? n : void 0;
+  }
+  return void 0;
+}
+
+// src/normalizer/transformers/openinference/index.ts
+var Attrs2 = {
+  SPAN_KIND: "openinference.span.kind",
+  MODEL: "llm.model_name",
+  PROVIDER: "llm.provider",
+  SYSTEM: "llm.system",
+  INVOCATION_PARAMETERS: "llm.invocation_parameters",
+  TOKEN_PROMPT: "llm.token_count.prompt",
+  TOKEN_COMPLETION: "llm.token_count.completion",
+  TOKEN_TOTAL: "llm.token_count.total",
+  TOKEN_REASONING: "llm.token_count.completion_details.reasoning",
+  INPUT_VALUE: "input.value",
+  INPUT_MIME: "input.mime_type",
+  OUTPUT_VALUE: "output.value",
+  OUTPUT_MIME: "output.mime_type",
+  TOOL_NAME: "tool.name",
+  SESSION_ID: "session.id",
+  USER_ID: "user.id",
+  METADATA: "metadata"
+};
+var INPUT_MESSAGES = {
+  prefix: "llm.input_messages",
+  messageInfix: "message.",
+  contentsKey: "contents",
+  contentsPartInfix: "message_content.",
+  toolCalls: {
+    arrayKey: "tool_calls",
+    infix: "tool_call.",
+    idKey: "id",
+    nameKey: "function.name",
+    argsKey: "function.arguments"
+  }
+};
+var OUTPUT_MESSAGES = {
+  ...INPUT_MESSAGES,
+  prefix: "llm.output_messages"
+};
+var RETRIEVAL_PREFIX = "retrieval.documents";
+function extractSettings(attributes) {
+  var _a, _b, _c, _d, _e;
+  const raw = attributes[Attrs2.INVOCATION_PARAMETERS];
+  if (raw === void 0) return void 0;
+  let parsed;
+  if (typeof raw === "string") {
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      return void 0;
+    }
+  } else if (typeof raw === "object" && raw !== null) {
+    parsed = raw;
+  } else {
+    return void 0;
+  }
+  if (!parsed || typeof parsed !== "object") return void 0;
+  const settings = {};
+  const temperature = toNumber(parsed.temperature);
+  const maxTokens = toNumber((_b = (_a = parsed.max_tokens) != null ? _a : parsed.maxTokens) != null ? _b : parsed.max_completion_tokens);
+  const topP = toNumber((_c = parsed.top_p) != null ? _c : parsed.topP);
+  const presencePenalty = toNumber((_d = parsed.presence_penalty) != null ? _d : parsed.presencePenalty);
+  const frequencyPenalty = toNumber((_e = parsed.frequency_penalty) != null ? _e : parsed.frequencyPenalty);
+  if (temperature !== void 0) settings.temperature = temperature;
+  if (maxTokens !== void 0) settings.maxTokens = maxTokens;
+  if (topP !== void 0) settings.topP = topP;
+  if (presencePenalty !== void 0) settings.presencePenalty = presencePenalty;
+  if (frequencyPenalty !== void 0) settings.frequencyPenalty = frequencyPenalty;
+  return Object.keys(settings).length > 0 ? settings : void 0;
+}
+function extractGenericInput(attributes) {
+  const value = attributes[Attrs2.INPUT_VALUE];
+  if (value === void 0) return {};
+  const mime = attributes[Attrs2.INPUT_MIME];
+  if (mime === "application/json" && typeof value === "string") {
+    try {
+      const parsed = JSON.parse(value);
+      if (Array.isArray(parsed) && parsed.every((m) => m && typeof m === "object" && "role" in m)) {
+        return { input: parsed };
+      }
+      return { input: [{ role: "user", content: JSON.stringify(parsed) }] };
+    } catch {
+    }
+  }
+  return { input: [{ role: "user", content: String(value) }] };
+}
+function extractGenericOutput(attributes) {
+  const value = attributes[Attrs2.OUTPUT_VALUE];
+  if (value === void 0) return {};
+  const mime = attributes[Attrs2.OUTPUT_MIME];
+  if (mime === "application/json" && typeof value === "string") {
+    try {
+      const parsed = JSON.parse(value);
+      return { output: JSON.stringify(parsed), outputObject: parsed };
+    } catch {
+    }
+  }
+  return { output: String(value) };
+}
+function extractRetrievalDocuments(attributes) {
+  const docs = [];
+  for (const i of collectIndices(attributes, RETRIEVAL_PREFIX)) {
+    const content = attributes[`${RETRIEVAL_PREFIX}.${i}.document.content`];
+    if (typeof content === "string" && content.length > 0) docs.push(content);
+  }
+  return docs.length > 0 ? docs.join("\n\n") : void 0;
+}
+var OpenInferenceTransformer = class {
+  classify(_span, attributes) {
+    const kind = attributes[Attrs2.SPAN_KIND];
+    if (typeof kind === "string" && kind.toUpperCase() === "LLM") return "GENERATION" /* GENERATION */;
+    if (attributes[Attrs2.TOKEN_PROMPT] !== void 0 || attributes[Attrs2.TOKEN_COMPLETION] !== void 0) {
+      return "GENERATION" /* GENERATION */;
+    }
+    return "SPAN" /* SPAN */;
+  }
+  transform(_span, attributes) {
+    const result = {};
+    const model = attributes[Attrs2.MODEL];
+    if (typeof model === "string" && model.length > 0) result.model = model;
+    const inputTokens = toNumber(attributes[Attrs2.TOKEN_PROMPT]);
+    const outputTokens = toNumber(attributes[Attrs2.TOKEN_COMPLETION]);
+    const totalTokens = toNumber(attributes[Attrs2.TOKEN_TOTAL]);
+    const reasoningTokens = toNumber(attributes[Attrs2.TOKEN_REASONING]);
+    if (inputTokens !== void 0) result.inputTokens = inputTokens;
+    if (outputTokens !== void 0) result.outputTokens = outputTokens;
+    if (totalTokens !== void 0) {
+      result.totalTokens = totalTokens;
+    } else if (inputTokens !== void 0 && outputTokens !== void 0) {
+      result.totalTokens = inputTokens + outputTokens;
+    }
+    if (reasoningTokens !== void 0) result.reasoningTokens = reasoningTokens;
+    const settings = extractSettings(attributes);
+    if (settings) result.settings = settings;
+    const inputMessages = parseIndexedMessages(attributes, INPUT_MESSAGES);
+    if (inputMessages) {
+      result.input = inputMessages;
+    } else {
+      Object.assign(result, extractGenericInput(attributes));
+    }
+    const outputMessages = parseIndexedMessages(attributes, OUTPUT_MESSAGES);
+    const toolCalls = extractIndexedToolCalls(attributes, OUTPUT_MESSAGES);
+    if (toolCalls) result.toolCalls = toolCalls;
+    if (outputMessages) {
+      const text = messagesToPlainText(outputMessages);
+      if (text) result.output = text;
+    } else {
+      Object.assign(result, extractGenericOutput(attributes));
+    }
+    if (result.output === void 0) {
+      const docs = extractRetrievalDocuments(attributes);
+      if (docs) result.output = docs;
+    }
+    const toolName = attributes[Attrs2.TOOL_NAME];
+    if (typeof toolName === "string" && toolName.length > 0) result.name = toolName;
+    const sessionId = attributes[Attrs2.SESSION_ID];
+    if (sessionId !== void 0) result.sessionId = String(sessionId);
+    const userId = attributes[Attrs2.USER_ID];
+    if (userId !== void 0) result.userId = String(userId);
+    const metadataRaw = attributes[Attrs2.METADATA];
+    if (metadataRaw !== void 0) {
+      let parsed = metadataRaw;
+      if (typeof metadataRaw === "string") {
+        try {
+          parsed = JSON.parse(metadataRaw);
+        } catch {
+          parsed = void 0;
+        }
+      }
+      if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+        const metadata = {};
+        for (const [k, v] of Object.entries(parsed)) {
+          metadata[k] = typeof v === "string" ? v : JSON.stringify(v);
+        }
+        if (Object.keys(metadata).length > 0) result.metadata = metadata;
+      }
+    }
+    return result;
+  }
+};
+
+// src/normalizer/transformers/openllmetry/index.ts
+var Attrs3 = {
+  SPAN_KIND: "traceloop.span.kind",
+  REQUEST_MODEL: "gen_ai.request.model",
+  RESPONSE_MODEL: "gen_ai.response.model",
+  REQUEST_TEMPERATURE: "gen_ai.request.temperature",
+  REQUEST_MAX_TOKENS: "gen_ai.request.max_tokens",
+  REQUEST_TOP_P: "gen_ai.request.top_p",
+  REQUEST_PRESENCE_PENALTY: "gen_ai.request.presence_penalty",
+  REQUEST_FREQUENCY_PENALTY: "gen_ai.request.frequency_penalty",
+  RESPONSE_FINISH_REASON: "gen_ai.response.finish_reason",
+  TOTAL_TOKENS_ALT: "llm.usage.total_tokens",
+  ENTITY_INPUT: "traceloop.entity.input",
+  ENTITY_OUTPUT: "traceloop.entity.output",
+  ENTITY_NAME: "traceloop.entity.name",
+  ASSOCIATION_PREFIX: "traceloop.association.properties."
+};
+var PROMPT_MESSAGES = {
+  prefix: "gen_ai.prompt",
+  messageInfix: "",
+  toolCalls: { arrayKey: "tool_calls", infix: "", idKey: "id", nameKey: "name", argsKey: "arguments" }
+};
+var COMPLETION_MESSAGES = {
+  ...PROMPT_MESSAGES,
+  prefix: "gen_ai.completion"
+};
+function extractSettings2(attributes) {
+  const settings = {};
+  const temperature = toNumber(attributes[Attrs3.REQUEST_TEMPERATURE]);
+  const maxTokens = toNumber(attributes[Attrs3.REQUEST_MAX_TOKENS]);
+  const topP = toNumber(attributes[Attrs3.REQUEST_TOP_P]);
+  const presencePenalty = toNumber(attributes[Attrs3.REQUEST_PRESENCE_PENALTY]);
+  const frequencyPenalty = toNumber(attributes[Attrs3.REQUEST_FREQUENCY_PENALTY]);
+  if (temperature !== void 0) settings.temperature = temperature;
+  if (maxTokens !== void 0) settings.maxTokens = maxTokens;
+  if (topP !== void 0) settings.topP = topP;
+  if (presencePenalty !== void 0) settings.presencePenalty = presencePenalty;
+  if (frequencyPenalty !== void 0) settings.frequencyPenalty = frequencyPenalty;
+  return Object.keys(settings).length > 0 ? settings : void 0;
+}
+function extractEntityInput(attributes) {
+  const raw = attributes[Attrs3.ENTITY_INPUT];
+  if (raw === void 0) return void 0;
+  const text = typeof raw === "string" ? raw : JSON.stringify(raw);
+  return [{ role: "user", content: text }];
+}
+function extractEntityOutput(attributes) {
+  const raw = attributes[Attrs3.ENTITY_OUTPUT];
+  if (raw === void 0) return {};
+  if (typeof raw === "string") {
+    try {
+      const parsed = JSON.parse(raw);
+      if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+        return { output: raw, outputObject: parsed };
+      }
+    } catch {
+    }
+    return { output: raw };
+  }
+  const result = { output: JSON.stringify(raw) };
+  if (typeof raw === "object" && raw !== null) result.outputObject = raw;
+  return result;
+}
+function extractFromEvents(events, attrKey) {
+  var _a, _b, _c;
+  if (!events) return void 0;
+  for (const event of events) {
+    const value = (_c = (_a = event.attributes) == null ? void 0 : _a[attrKey]) != null ? _c : (_b = event.attributes) == null ? void 0 : _b["content"];
+    if (typeof value === "string" && value.length > 0) return value;
+  }
+  return void 0;
+}
+function extractFinishReason(attributes) {
+  const direct = attributes[Attrs3.RESPONSE_FINISH_REASON];
+  if (direct !== void 0) {
+    return Array.isArray(direct) ? String(direct[0]) : String(direct);
+  }
+  const indexed = attributes["gen_ai.completion.0.finish_reason"];
+  if (indexed !== void 0) return String(indexed);
+  return void 0;
+}
+var OpenLLMetryTransformer = class {
+  classify(_span, attributes) {
+    const kind = attributes[Attrs3.SPAN_KIND];
+    if (typeof kind === "string" && kind.toUpperCase() === "LLM") return "GENERATION" /* GENERATION */;
+    if (attributes["gen_ai.usage.prompt_tokens"] !== void 0 || attributes["gen_ai.usage.completion_tokens"] !== void 0 || attributes["gen_ai.usage.input_tokens"] !== void 0) {
+      return "GENERATION" /* GENERATION */;
+    }
+    return "SPAN" /* SPAN */;
+  }
+  transform(span, attributes) {
+    var _a;
+    const result = {};
+    const model = (_a = attributes[Attrs3.RESPONSE_MODEL]) != null ? _a : attributes[Attrs3.REQUEST_MODEL];
+    if (typeof model === "string" && model.length > 0) result.model = model;
+    const tokens = parseTokens(attributes, {
+      inputKey: "gen_ai.usage.input_tokens",
+      outputKey: "gen_ai.usage.output_tokens",
+      totalKey: "gen_ai.usage.total_tokens",
+      promptKey: "gen_ai.usage.prompt_tokens",
+      completionKey: "gen_ai.usage.completion_tokens"
+    });
+    if (tokens.inputTokens !== void 0) result.inputTokens = tokens.inputTokens;
+    if (tokens.outputTokens !== void 0) result.outputTokens = tokens.outputTokens;
+    if (tokens.totalTokens !== void 0) {
+      result.totalTokens = tokens.totalTokens;
+    } else {
+      const altTotal = toNumber(attributes[Attrs3.TOTAL_TOKENS_ALT]);
+      if (altTotal !== void 0) result.totalTokens = altTotal;
+    }
+    const settings = extractSettings2(attributes);
+    if (settings) result.settings = settings;
+    const finishReason = extractFinishReason(attributes);
+    if (finishReason) result.finishReason = finishReason;
+    const promptMessages = parseIndexedMessages(attributes, PROMPT_MESSAGES);
+    if (promptMessages) {
+      result.input = promptMessages;
+    } else {
+      const entityInput = extractEntityInput(attributes);
+      if (entityInput) {
+        result.input = entityInput;
+      } else {
+        const eventPrompt = extractFromEvents(span.events, "gen_ai.prompt");
+        if (eventPrompt) result.input = [{ role: "user", content: eventPrompt }];
+      }
+    }
+    const completionMessages = parseIndexedMessages(attributes, COMPLETION_MESSAGES);
+    const toolCalls = extractIndexedToolCalls(attributes, COMPLETION_MESSAGES);
+    if (toolCalls) result.toolCalls = toolCalls;
+    if (completionMessages) {
+      const text = messagesToPlainText(completionMessages);
+      if (text) result.output = text;
+    } else {
+      const entityOutput = extractEntityOutput(attributes);
+      if (entityOutput.output !== void 0) {
+        Object.assign(result, entityOutput);
+      } else {
+        const eventCompletion = extractFromEvents(span.events, "gen_ai.completion");
+        if (eventCompletion) result.output = eventCompletion;
+      }
+    }
+    const entityName = attributes[Attrs3.ENTITY_NAME];
+    if (typeof entityName === "string" && entityName.length > 0) result.traceName = entityName;
+    const metadata = {};
+    for (const [key, value] of Object.entries(attributes)) {
+      if (!key.startsWith(Attrs3.ASSOCIATION_PREFIX)) continue;
+      const prop = key.slice(Attrs3.ASSOCIATION_PREFIX.length);
+      if (prop === "session_id") {
+        result.sessionId = String(value);
+      } else if (prop === "user_id") {
+        result.userId = String(value);
+      } else if (prop) {
+        metadata[prop] = typeof value === "string" ? value : JSON.stringify(value);
+      }
+    }
+    if (Object.keys(metadata).length > 0) result.metadata = metadata;
+    return result;
+  }
+};
+
+// src/normalizer/transformers/dispatching/index.ts
+var OPENINFERENCE_INDEXED = /^llm\.(input_messages|output_messages|token_count)\./;
+var OPENLLMETRY_INDEXED = /^gen_ai\.(prompt|completion)\.\d+\./;
+function isOpenInference(attributes) {
+  if (attributes["openinference.span.kind"] !== void 0) return true;
+  if (attributes["llm.model_name"] !== void 0) return true;
+  for (const key of Object.keys(attributes)) {
+    if (OPENINFERENCE_INDEXED.test(key)) return true;
+  }
+  return false;
+}
+function isOpenLLMetry(attributes) {
+  for (const key of Object.keys(attributes)) {
+    if (key.startsWith("traceloop.")) return true;
+    if (OPENLLMETRY_INDEXED.test(key)) return true;
+  }
+  return false;
+}
+var DispatchingTransformer = class {
+  constructor() {
+    this.openInference = new OpenInferenceTransformer();
+    this.openLLMetry = new OpenLLMetryTransformer();
+    this.otelGenAi = new OtelGenAiTransformer();
+  }
+  /** Choose the extractor for a span from its attribute signature. OpenInference
+   *  is checked before OpenLLMetry because its markers (`llm.*`,
+   *  `openinference.span.kind`) are more specific; the bare OTel GenAI
+   *  transformer is the catch-all. */
+  select(attributes) {
+    if (isOpenInference(attributes)) return this.openInference;
+    if (isOpenLLMetry(attributes)) return this.openLLMetry;
+    return this.otelGenAi;
+  }
+  classify(span, attributes) {
+    return this.select(attributes).classify(span, attributes);
+  }
+  transform(span, attributes) {
+    return this.select(attributes).transform(span, attributes);
+  }
+};
+
 // src/normalizer/converters/otlp-converter.ts
 function convertOtlpValue(value) {
   var _a;
@@ -2325,7 +2839,7 @@ registry.register("ai", new AiSdkTransformer());
 registry.register("default-tracer", new MastraTransformer());
 registry.register("agentmark", new AgentMarkTransformer());
 registry.register("pydantic-ai", new OtelGenAiTransformer());
-registry.setDefault(new OtelGenAiTransformer());
+registry.setDefault(new DispatchingTransformer());
 var OTLP_STATUS_CODE_MAP = {
   "0": "0",
   STATUS_CODE_UNSET: "0",
@@ -2448,27 +2962,34 @@ export {
   AgentMarkTransformer,
   AiSdkTransformer,
   AgentMarkTransformer as ClaudeAgentTransformer,
+  DispatchingTransformer,
   MastraTransformer,
+  OpenInferenceTransformer,
+  OpenLLMetryTransformer,
   OtelGenAiTransformer,
   SEMANTIC_KINDS,
   SpanType,
   TransformerRegistry,
   TypeClassifier,
+  collectIndices,
   convertOtlpAttributes,
   createSignature,
   deriveTraceIO,
   detectVersion,
   extractCustomMetadata,
+  extractIndexedToolCalls,
   extractReasoningFromProviderMetadata,
   extractResourceScopeSpan,
   fetchPromptsFrontmatter,
   findPromptFiles,
   generateTypeDefinitions,
   generateUnique8CharString,
+  messagesToPlainText,
   normalizeOtlpSpans,
   normalizeOtlpStatusCode,
   normalizeSpan,
   parseAgentMarkAttributes,
+  parseIndexedMessages,
   parseMetadata,
   parseTokens,
   registry,