@ganju/utils 0.0.9 → 0.0.11

package/dist/toolConfirmation.js

@@ -0,0 +1,217 @@
+"use strict";
+// When an organization turns on "confirm sensitive actions", a channel bot that
+// wants to run a sensitive tool asks first, and the call runs only if the
+// participant's next message is a plain yes. These are the pieces of that rule
+// that aren't the runner's own bookkeeping.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatConfirmationSummary = exports.isConfirmationReply = exports.parseConfirmationButton = exports.confirmationButtonId = exports.confirmationButtonLabels = exports.languageFromPhoneNumber = exports.annotationsForEffect = exports.TOOL_EFFECTS = void 0;
+// What an owner says a tool they wrote does, in the words the dashboard asks
+// it in, and the MCP annotations each answer stands for: `read` only looks
+// things up, `write` changes things that can be undone, `sensitive` sends,
+// deletes, charges or can't be undone — the kind a confirming channel asks
+// about.
+exports.TOOL_EFFECTS = ['read', 'write', 'sensitive'];
+const annotationsForEffect = (effect) => effect === 'read'
+    ? { readOnlyHint: true, destructiveHint: false }
+    : effect === 'write'
+        ? { readOnlyHint: false, destructiveHint: false }
+        : { readOnlyHint: false, destructiveHint: true };
+exports.annotationsForEffect = annotationsForEffect;
+// Whole replies that mean yes, lowercased with accents and punctuation
+// removed. English, Spanish and Portuguese, the languages bots here answer in.
+const AFFIRMATIVE = new Set([
+    'y',
+    'yes',
+    'yeah',
+    'yep',
+    'yup',
+    'sure',
+    'ok',
+    'okay',
+    'confirm',
+    'confirmed',
+    'approve',
+    'approved',
+    'go',
+    'go ahead',
+    'do it',
+    'send',
+    'send it',
+    'proceed',
+    'yes please',
+    'yes send it',
+    'yes do it',
+    'yes go ahead',
+    'yes confirm',
+    'sure thing',
+    'go for it',
+    'sounds good',
+    'looks good',
+    'all good',
+    'correct',
+    'si hazlo',
+    'si adelante',
+    'si envialo ya',
+    'perfecto',
+    'correcto',
+    'esta bien',
+    'ok dale',
+    'ok envialo',
+    'si',
+    'sip',
+    'claro',
+    'dale',
+    'va',
+    'vale',
+    'listo',
+    'confirmo',
+    'confirmado',
+    'confirmar',
+    'adelante',
+    'hazlo',
+    'envialo',
+    'enviar',
+    'de acuerdo',
+    'si por favor',
+    'si claro',
+    'si dale',
+    'si envialo',
+    'si confirmo',
+    'sim',
+    'pode',
+    'pode enviar'
+]);
+const AFFIRMATIVE_EMOJI = new Set(['👍', '✅', '👌', '🆗']);
+// The Yes/No buttons offered under a confirmation question, in the
+// participant's language when the platform reports it. The labels are also
+// what a tap sends where a button is just a canned reply (Telegram's reply
+// keyboard), so "yes" must read as a yes to isConfirmationReply — and it does,
+// in every language here.
+const BUTTON_LABELS = {
+    en: { yes: '✅ Yes', no: '❌ No' },
+    es: { yes: '✅ Sí', no: '❌ No' },
+    pt: { yes: '✅ Sim', no: '❌ Não' }
+};
+// WhatsApp reports no locale, so a button's language comes from the calling
+// code of the participant's number: the Spanish- and Portuguese-speaking
+// countries, English otherwise. Longest prefix wins.
+const CALLING_CODE_LANGUAGE = {
+    '34': 'es',
+    '51': 'es',
+    '52': 'es',
+    '53': 'es',
+    '54': 'es',
+    '56': 'es',
+    '57': 'es',
+    '58': 'es',
+    '240': 'es',
+    '502': 'es',
+    '503': 'es',
+    '504': 'es',
+    '505': 'es',
+    '506': 'es',
+    '507': 'es',
+    '591': 'es',
+    '593': 'es',
+    '595': 'es',
+    '598': 'es',
+    '1787': 'es',
+    '1809': 'es',
+    '1829': 'es',
+    '1849': 'es',
+    '55': 'pt',
+    '238': 'pt',
+    '244': 'pt',
+    '245': 'pt',
+    '258': 'pt',
+    '351': 'pt'
+};
+const languageFromPhoneNumber = (phone) => {
+    const digits = phone.replace(/\D/g, '');
+    for (let length = 4; length >= 2; length--) {
+        const language = CALLING_CODE_LANGUAGE[digits.slice(0, length)];
+        if (language)
+            return language;
+    }
+    return null;
+};
+exports.languageFromPhoneNumber = languageFromPhoneNumber;
+const confirmationButtonLabels = (locale) => {
+    const language = (locale || '').toLowerCase().split(/[-_]/)[0];
+    return BUTTON_LABELS[language] ?? BUTTON_LABELS.en;
+};
+exports.confirmationButtonLabels = confirmationButtonLabels;
+// Where the platform sends back a button's id rather than its label (Discord,
+// WhatsApp), the id says which answer it is and who was asked: in a shared
+// channel everyone sees the buttons, and only that participant's tap counts.
+// The runner never sees the id — a tap reaches it as the text "yes" or "no".
+const CONFIRMATION_BUTTON_PREFIX = 'ganju-confirm';
+const confirmationButtonId = (answer, externalParticipantId) => `${CONFIRMATION_BUTTON_PREFIX}:${answer}:${externalParticipantId}`;
+exports.confirmationButtonId = confirmationButtonId;
+const parseConfirmationButton = (id) => {
+    const [prefix, answer, ...rest] = id.split(':');
+    const externalParticipantId = rest.join(':');
+    if (prefix !== CONFIRMATION_BUTTON_PREFIX || !externalParticipantId) {
+        return null;
+    }
+    if (answer !== 'yes' && answer !== 'no')
+        return null;
+    return { text: answer, externalParticipantId };
+};
+exports.parseConfirmationButton = parseConfirmationButton;
+const normalize = (text) => text
+    .toLowerCase()
+    .normalize('NFKD')
+    .replace(/[̀-ͯ]/g, '')
+    .replace(/[^a-z0-9\s]+/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+/**
+ * Whether a reply confirms the pending action. Deliberately narrow: the whole
+ * message has to be a yes. "Yes, but send it to Ana instead" is not one — it
+ * asks for a different action, and running the original would do exactly
+ * what the person just said not to. Anything that isn't a clear yes cancels,
+ * so the worst a misread costs is asking again.
+ */
+const isConfirmationReply = (text) => {
+    const trimmed = text.trim();
+    if (AFFIRMATIVE_EMOJI.has(trimmed))
+        return true;
+    return AFFIRMATIVE.has(normalize(trimmed));
+};
+exports.isConfirmationReply = isConfirmationReply;
+// How much of each argument the summary shows, and how many arguments.
+const SUMMARY_VALUE_MAX = 200;
+const SUMMARY_ARGUMENTS_MAX = 10;
+const summaryValue = (value) => {
+    const text = typeof value === 'string'
+        ? value
+        : (JSON.stringify(value) ?? String(value));
+    // One line, and no backticks, so a value can neither fake another line of
+    // the summary nor break out of its code span into formatting or links.
+    const flat = text.replace(/\s+/g, ' ').replace(/`/g, "'").trim();
+    return flat.length > SUMMARY_VALUE_MAX
+        ? `${flat.slice(0, SUMMARY_VALUE_MAX)}…`
+        : flat;
+};
+/**
+ * What the held calls will do, written from the calls themselves and appended
+ * under the model's question. The model words the question, but a yes runs the
+ * stored calls — so the participant has to see those, not only the model's
+ * account of them, which a prompt injection earlier in the turn could have
+ * bent.
+ */
+const formatConfirmationSummary = (calls, titleOf) => calls
+    .map(call => {
+    const entries = Object.entries(call.arguments ?? {}).filter(([, value]) => value !== undefined && value !== null && value !== '');
+    const lines = entries
+        .slice(0, SUMMARY_ARGUMENTS_MAX)
+        .map(([key, value]) => `- ${key}: \`${summaryValue(value)}\``);
+    if (entries.length > SUMMARY_ARGUMENTS_MAX)
+        lines.push('- …');
+    // The hourglass says "not done yet" in every language, whatever the
+    // model's own words above it claim.
+    return [`⏳ **${titleOf(call.name)}**`, ...lines].join('\n');
+})
+    .join('\n\n');
+exports.formatConfirmationSummary = formatConfirmationSummary;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ganju/utils",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "description": "Shared constants, schemas and helpers for Ganju",
   "license": "Apache-2.0",
   "author": "MontoyaAndres <andresmontoyafcb@gmail.com>",

package/src/channelDebounce.ts

@@ -12,8 +12,41 @@ export interface BufferedChannelMessage {
   text: string;
   externalMessageId: string | null;
   receivedAt: number;
+  // Answer without waiting for the burst to settle: an explicit command, or a
+  // message on a channel with buffering turned off. It still goes through the
+  // buffer — that is what runs the turn under a caller patient enough for a
+  // long answer — it just flushes at once, taking any pending text with it.
+  immediate?: boolean;
+  // The prompt this message invoked, when it was a slash command.
+  command?: BufferedChannelCommand | null;
+  // Where this message must be answered, when that isn't the envelope's
+  // `delivery` — a Discord interaction waiting on its deferred response. Kept
+  // on the message because the envelope is replaced by every newer push, and a
+  // plain message typed a moment later must not orphan the interaction.
+  reply?: Record<string, unknown> | null;
 }
 
+// A resolved slash-command prompt, carried with the message that invoked it so
+// the flush can run it exactly as the webhook would have.
+export interface BufferedChannelCommand {
+  promptId: string;
+  artifactPromptId: string | null;
+  promptTitle: string;
+  args: Record<string, string>;
+}
+
+// The prompt a batch runs: the newest command in it, since that is the one the
+// participant asked for last. Null for a batch of plain text.
+export const batchCommand = (
+  messages: BufferedChannelMessage[]
+): BufferedChannelCommand | null => {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const command = messages[i].command;
+    if (command) return command;
+  }
+  return null;
+};
+
 // Everything the flush needs to run the turn and deliver the reply, minus the
 // message texts themselves. The newest push wins: a display name or thread id
 // can change mid-burst, and the freshest is the one to answer against.
@@ -35,6 +68,9 @@ export interface ChannelBufferEnvelope {
 export interface ChannelBufferFlush {
   envelope: ChannelBufferEnvelope;
   messages: BufferedChannelMessage[];
+  // Stable across every hand-off of the same batch, so the worker can tell a
+  // retry of a turn it already answered from a new one.
+  batchId: string;
 }
 
 // Read the channel's debounce window off its `config`, clamped to the supported

package/src/chunking.ts

@@ -1,4 +1,5 @@
 import { constants } from './constants';
+import { decodeEntities } from './sanitize';
 
 import type {
   ExtractedDocument,
@@ -50,24 +51,140 @@ export const splitRecursive = (
   return chunks;
 };
 
-export const chunkText = (
+// Each piece keeps its `raw` text — the slice of the input it covers, without
+// the overlap borrowed from the piece before — so a caller can walk the input
+// in order without reading any of it twice.
+const chunkPieces = (
   text: string,
   options?: { targetChars?: number; overlapChars?: number }
-): string[] => {
+): Array<{ raw: string; content: string }> => {
   const target = options?.targetChars ?? constants.CHUNK_TARGET_CHARS;
   const overlap = options?.overlapChars ?? constants.CHUNK_OVERLAP_CHARS;
 
   const trimmed = text.trim();
   if (!trimmed) return [];
-  if (trimmed.length <= target) return [trimmed];
+  if (trimmed.length <= target) return [{ raw: trimmed, content: trimmed }];
 
   const rawChunks = splitRecursive(trimmed, target);
-  const withOverlap: string[] = [];
+  const pieces: Array<{ raw: string; content: string }> = [];
   for (let i = 0; i < rawChunks.length; i++) {
     const prev = i > 0 ? rawChunks[i - 1].slice(-overlap) : '';
-    withOverlap.push((prev ? prev + '\n' : '') + rawChunks[i]);
+    pieces.push({
+      raw: rawChunks[i],
+      content: (prev ? prev + '\n' : '') + rawChunks[i]
+    });
+  }
+  return pieces.filter(p => p.content.trim().length > 0);
+};
+
+export const chunkText = (
+  text: string,
+  options?: { targetChars?: number; overlapChars?: number }
+): string[] => chunkPieces(text, options).map(p => p.content);
+
+export type HeadingSyntax = 'markdown' | 'html';
+
+interface HeadingEvent {
+  offset: number;
+  level: number;
+  text: string;
+}
+
+const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
+const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
+const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
+const MAX_HEADING_CHARS = 120;
+
+const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
+
+/**
+ * Which heading syntax a document is written in, or null when it has none we
+ * can trust. Plain text is left out on purpose: a line that happens to start
+ * with "# " there is as likely a comment or a list as a title.
+ */
+export const headingSyntaxFor = (input: {
+  mimeType?: string | null;
+  fileName?: string | null;
+  isWebPage?: boolean;
+}): HeadingSyntax | null => {
+  // Crawled pages are flattened to text with their headings written back as
+  // Markdown lines, whatever the page's own content type was.
+  if (input.isWebPage) return 'markdown';
+  if (input.mimeType === constants.MIMETYPE_TEXT_MARKDOWN) return 'markdown';
+  if (input.mimeType === constants.MIMETYPE_TEXT_HTML) return 'html';
+  if (input.fileName && MARKDOWN_FILE_NAME.test(input.fileName)) {
+    return 'markdown';
+  }
+  return null;
+};
+
+const cleanHeading = (text: string): string =>
+  text.replace(/\s+/g, ' ').trim().slice(0, MAX_HEADING_CHARS);
+
+const scanMarkdownHeadings = (
+  text: string,
+  state: { inFence: boolean }
+): HeadingEvent[] => {
+  const events: HeadingEvent[] = [];
+  let offset = 0;
+  for (const line of text.split('\n')) {
+    // A `#` inside a fenced block is a shell comment or a preprocessor line,
+    // not a title — and a fence can open in one chunk and close in the next.
+    if (MARKDOWN_FENCE.test(line)) {
+      state.inFence = !state.inFence;
+    } else if (!state.inFence) {
+      const match = MARKDOWN_HEADING.exec(line);
+      const heading = match ? cleanHeading(match[2]) : '';
+      if (match && heading) {
+        events.push({ offset, level: match[1].length, text: heading });
+      }
+    }
+    offset += line.length + 1;
   }
-  return withOverlap.filter(c => c.trim().length > 0);
+  return events;
+};
+
+const scanHtmlHeadings = (text: string): HeadingEvent[] =>
+  [...text.matchAll(HTML_HEADING)]
+    .map(match => ({
+      offset: match.index ?? 0,
+      level: Number(match[1]),
+      text: cleanHeading(decodeEntities(match[2].replace(/<[^>]*>/g, ' ')))
+    }))
+    .filter(event => event.text);
+
+/**
+ * Follows the heading hierarchy across a document's chunks, fed in order.
+ * Returns, for each chunk, the path of headings it sits under — the section a
+ * reader would be pointed to. A chunk that opens with a heading belongs to that
+ * heading; one that starts mid-section belongs to the section it continues,
+ * even when a later heading begins inside it.
+ */
+const createHeadingTracker = (syntax: HeadingSyntax) => {
+  let stack: Array<{ level: number; text: string }> = [];
+  const fence = { inFence: false };
+  const apply = (event: HeadingEvent) => {
+    stack = stack.filter(h => h.level < event.level);
+    stack.push({ level: event.level, text: event.text });
+  };
+
+  return (raw: string): string[] => {
+    const events =
+      syntax === 'markdown'
+        ? scanMarkdownHeadings(raw, fence)
+        : scanHtmlHeadings(raw);
+    const [first, ...rest] = events;
+    // Before a document's first heading there is no section to continue — only
+    // the metadata header, or the markup around an HTML body — so a chunk that
+    // holds the first heading belongs to it wherever in the chunk it falls.
+    const opensWithHeading =
+      first !== undefined &&
+      (stack.length === 0 || raw.slice(0, first.offset).trim() === '');
+    if (opensWithHeading) apply(first);
+    const path = stack.map(h => h.text);
+    for (const event of opensWithHeading ? rest : events) apply(event);
+    return path;
+  };
 };
 
 export const buildHeader = (input: {
@@ -87,9 +204,14 @@ export const buildHeader = (input: {
     .filter(Boolean)
     .join('\n');
 
-export interface ChunkMetadata extends ExtractedDocumentMetadata {
-  chunkIndexInPage: number;
-  chunksInPage: number;
+// Loose on purpose: a chunk of inline text has no extracted document behind it,
+// so only `headingPath` can be known about it.
+export interface ChunkMetadata extends Partial<ExtractedDocumentMetadata> {
+  chunkIndexInPage?: number;
+  chunksInPage?: number;
+  // The headings this chunk sits under, outermost first. Present only for
+  // Markdown and HTML, and only when the chunk is under at least one heading.
+  headingPath?: string[];
 }
 
 export interface PreparedChunk {
@@ -97,28 +219,43 @@ export interface PreparedChunk {
   metadata: ChunkMetadata | null;
 }
 
+const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g;
+
 export const prepareChunks = (
   header: string,
   documents: ExtractedDocument[] | null,
   fallbackContent: string | null,
-  options?: { targetChars?: number; overlapChars?: number }
+  options?: {
+    targetChars?: number;
+    overlapChars?: number;
+    // The resource's own type and file name, which decide the heading syntax
+    // for inline content and for documents that don't carry their own.
+    mimeType?: string | null;
+    fileName?: string | null;
+  }
 ): PreparedChunk[] => {
   if (documents && documents.length > 0) {
     const out: PreparedChunk[] = [];
     documents.forEach((doc, docIndex) => {
-      const body = (doc.pageContent || '')
-        .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
-        .trim();
+      const body = (doc.pageContent || '').replace(CONTROL_CHARS, '').trim();
       if (!body) return;
       const text = docIndex === 0 && header ? `${header}\n\n${body}` : body;
-      const pieces = chunkText(text, options);
+      const syntax = headingSyntaxFor({
+        mimeType: doc.metadata?.source?.mimeType ?? options?.mimeType,
+        fileName: doc.metadata?.source?.fileName ?? options?.fileName,
+        isWebPage: !!doc.metadata?.web
+      });
+      const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
+      const pieces = chunkPieces(text, options);
       pieces.forEach((piece, pieceIndex) => {
+        const headingPath = headingPathOf?.(piece.raw) ?? [];
         out.push({
-          content: piece,
+          content: piece.content,
           metadata: {
             ...doc.metadata,
             chunkIndexInPage: pieceIndex,
-            chunksInPage: pieces.length
+            chunksInPage: pieces.length,
+            ...(headingPath.length > 0 ? { headingPath } : {})
           }
         });
       });
@@ -126,12 +263,18 @@ export const prepareChunks = (
     return out;
   }
 
-  const body = (fallbackContent || '')
-    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
-    .trim();
+  const body = (fallbackContent || '').replace(CONTROL_CHARS, '').trim();
   const fullText = [header, body].filter(Boolean).join('\n\n');
-  return chunkText(fullText, options).map(content => ({
-    content,
-    metadata: null
-  }));
+  const syntax = headingSyntaxFor({
+    mimeType: options?.mimeType,
+    fileName: options?.fileName
+  });
+  const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
+  return chunkPieces(fullText, options).map(piece => {
+    const headingPath = headingPathOf?.(piece.raw) ?? [];
+    return {
+      content: piece.content,
+      metadata: headingPath.length > 0 ? { headingPath } : null
+    };
+  });
 };