@copilotkit/learning 0.0.1 → 1.80.0

package/src/__tests__/trajectory-collector.test.ts

@@ -134,6 +134,7 @@ describe("Trajectory capture contract", () => {
     const input = document.querySelector("input")!;
     input.value = "synthetic@example.com";
     input.dispatchEvent(new Event("input", { bubbles: true }));
+    vi.advanceTimersByTime(300);
     expect(send.mock.calls[0]?.[0].value).toMatchObject({
       route: "/deals",
       url: expect.stringContaining("?email="),
@@ -365,3 +366,107 @@ describe("Trajectory capture contract", () => {
     expect(vi.getTimerCount()).toBe(0);
   });
 });
+
+it.each(["custom", "click", "navigation", "stop", "pagehide"])(
+  "flushes pending text before %s",
+  (next) => {
+    const { collector, send } = setup();
+    document.body.innerHTML = '<input id="notes"><button>Approve</button>';
+    collector.start();
+    const input = document.querySelector("input")!;
+    input.value = "final notes";
+    input.dispatchEvent(new Event("input", { bubbles: true }));
+    expect(send.mock.calls.map(([event]) => event.name)).toEqual(["page"]);
+    if (next === "custom") collector.emit("order.approved", { total: 480 });
+    else if (next === "click") document.querySelector("button")!.click();
+    else if (next === "navigation") history.pushState(null, "", "/next");
+    else if (next === "pagehide") window.dispatchEvent(new Event("pagehide"));
+    else collector.stop();
+    const events = send.mock.calls.map(([event]) => event);
+    expect(events.map((event) => event.name)).toEqual([
+      "page",
+      "input",
+      ...(next === "custom"
+        ? ["order.approved"]
+        : next === "click" || next === "navigation"
+          ? [next]
+          : []),
+    ]);
+    expect(events[1]?.value).toMatchObject({
+      route: "/deals",
+      target: { value: "final notes" },
+    });
+    collector.stop();
+    vi.advanceTimersByTime(1000);
+    expect(send).toHaveBeenCalledTimes(events.length);
+  },
+);
+
+it("network capture does not flush or delay pending text in the trajectory collector", async () => {
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = vi.fn(async () => new Response(null, { status: 204 }));
+  const { collector, send } = setup();
+  document.body.innerHTML = "<input>";
+  try {
+    collector.start();
+    const input = document.querySelector("input")!;
+    input.value = "pending search";
+    input.dispatchEvent(new Event("input", { bubbles: true }));
+    await vi.advanceTimersByTimeAsync(100);
+
+    await fetch("/typeahead?q=pending");
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(send.mock.calls.map(([event]) => event.name)).toEqual([
+      "page",
+      "network",
+    ]);
+    await vi.advanceTimersByTimeAsync(199);
+    expect(send.mock.calls.map(([event]) => event.name)).toEqual([
+      "page",
+      "network",
+    ]);
+    await vi.advanceTimersByTimeAsync(1);
+    expect(send.mock.calls.map(([event]) => event.name)).toEqual([
+      "page",
+      "network",
+      "input",
+    ]);
+    expect(send.mock.calls[2]?.[0]).toMatchObject({
+      timestamp: NOW + 300,
+      value: { target: { value: "pending search" } },
+    });
+  } finally {
+    collector.stop();
+    globalThis.fetch = originalFetch;
+  }
+});
+
+it.each(["emit", "stop"])(
+  "a beforeSend stop/restart during %s cannot carry an old event into the new capture",
+  (action) => {
+    const { collector, send } = setup({
+      beforeSend(event) {
+        if (event.name === "input") {
+          collector.stop();
+          collector.start();
+        }
+        return event;
+      },
+    });
+    document.body.innerHTML = "<input>";
+    collector.start();
+    document.querySelector("input")!.value = "pending";
+    document
+      .querySelector("input")!
+      .dispatchEvent(new Event("input", { bubbles: true }));
+    if (action === "emit") collector.emit("old.action", {});
+    else collector.stop();
+    expect(send.mock.calls.map(([event]) => event.name)).toEqual([
+      "page",
+      "page",
+    ]);
+    collector.emit("new.action", {});
+    expect(send.mock.calls.at(-1)?.[0].name).toBe("new.action");
+  },
+);

package/src/clicks.ts

@@ -66,6 +66,298 @@ export function describeTarget(el: Element, redact: Redactor) {
   return described;
 }
 
+const MAX_LABEL_LENGTH = 80;
+// The text walk stops past these bounds. For a control's own text, that means no name.
+const MAX_WALK_NODES = 1000;
+const MAX_WALK_CHARS = 4 * MAX_LABEL_LENGTH;
+// Controls whose name comes from their content. An explicit role overrides the
+// tag, so `<li role="option">` qualifies and `<a href role="row">` does not.
+const CONTROL_ROLE =
+  /^(button|link|tab|menuitem(checkbox|radio)?|option|checkbox|switch|radio|treeitem)$/;
+const CONTENTEDITABLE = '[contenteditable]:not([contenteditable="false"])';
+const TEXT_ENTRY = `${CONTENTEDITABLE}, [role~=textbox], [role~=searchbox], [role~=combobox]`;
+const EDITABLE = `input, textarea, select, ${TEXT_ENTRY}`;
+// Card-like structure: a control containing it gets no name from its text,
+// except for one lone paragraph, or headings inside `summary`.
+const HEADING = "h1, h2, h3, h4, h5, h6, [role~=heading]";
+const STRUCTURE = `${HEADING}, p, ul, ol, li, dl, table, article, section, blockquote`;
+const SKIPPED = 'script, style, template, [hidden], [aria-hidden="true"]';
+// Inline elements that do not separate words.
+const PHRASING =
+  /^(b|i|u|em|strong|mark|small|sub|sup|abbr|code|kbd|s|q|cite|dfn|var|time|data|bdi|bdo|wbr|span)$/;
+const CAPTION_INPUT = /^(submit|button|reset|image)$/;
+// Input types that do not hold typed text.
+const NON_TEXT_INPUT =
+  /^(submit|button|reset|image|checkbox|radio|range|color|file|hidden)$/;
+// Includes bidi controls, zero-width characters, soft hyphens, word joiners,
+// variation selectors, and tag characters.
+const IGNORABLE = /\p{Default_Ignorable_Code_Point}/gu;
+// Quantifier bounds and run-start anchors keep each pattern linear. No
+// lookbehind: a regex literal with one fails to parse in older Safari.
+// A URL ends before trailing sentence punctuation.
+const URL_LIKE =
+  /(?:\b[a-z][\w+.-]{0,31}:\/\/|\bwww\.)[^\s<>"']*[^\s<>"'.,;:!?)]/gi;
+const EMAIL =
+  /[\p{L}\p{N}_.%+-]{1,64}@[\p{L}\p{N}-]{1,63}(?:\.[\p{L}\p{N}-]{1,63})+/gu;
+// JWTs, and three dotted segments that are each long or contain a digit, but
+// not dotted names such as `settings.notifications.preferences`.
+const JWT =
+  /(^|[^\w-])(?:eyJ[\w-]*\.[\w-]*\.[\w-]*|[\w-]{16,}\.[\w-]{16,}\.[\w-]{16,}|(?=[\w-]*\d)[\w-]{8,}\.(?=[\w-]*\d)[\w-]{8,}\.(?=[\w-]*\d)[\w-]{8,})/g;
+const TOKEN_RUN = /[\w+/=-]{24,}/g;
+
+const roleOf = (element: Element) =>
+  (element.getAttribute("role") ?? "").trim().split(/\s+/)[0]!.toLowerCase();
+
+const idRefs = (element: Element, name: string) =>
+  (element.getAttribute(name) ?? "").split(/\s+/).filter(Boolean);
+
+const rootOf = (element: Element) => {
+  const root = element.getRootNode();
+  return root instanceof Document || root instanceof ShadowRoot ? root : null;
+};
+
+/** Buttons, links with `href`, `summary`, and elements with a control role. */
+const isControl = (element: Element) => {
+  const role = roleOf(element);
+  return role
+    ? CONTROL_ROLE.test(role)
+    : element.matches("button, a[href], summary");
+};
+
+/** Text-like inputs, text areas, selects, and anything inside editable content or a text-entry role. */
+const holdsValue = (element: Element) =>
+  element instanceof HTMLInputElement
+    ? !NON_TEXT_INPUT.test(element.type)
+    : element.matches("textarea, select") ||
+      element.closest(TEXT_ENTRY) !== null;
+
+/**
+ * The name an image contributes: `alt` of an `img`, or `aria-label` (else a
+ * plain `<title>`) of an `svg` or `role="img"` element. `undefined` when the
+ * element is not an image.
+ */
+function imageName(element: Element) {
+  if (element.localName === "img") return element.getAttribute("alt");
+  if (element.localName !== "svg" && roleOf(element) !== "img")
+    return undefined;
+  const title = Array.from(element.children).find(
+    (child) => child.localName === "title" && child.childElementCount === 0,
+  );
+  return element.getAttribute("aria-label") ?? title?.textContent ?? null;
+}
+
+/**
+ * Rendered text. Inline formatting joins words; other elements separate them.
+ * Skips hidden subtrees, uses image names, and does
+ * not walk into images. Yields nothing for editable content or editable
+ * descendants. In `strict` mode (a control's own text) it also yields nothing
+ * past the walk bounds, for card-like structure, and for text that looks like
+ * separate fields: two `div`s that directly hold text, or three sibling
+ * elements that each carry text.
+ */
+function safeText(root: Element, strict: boolean) {
+  if (holdsValue(root)) return "";
+  let text = "";
+  let nodes = 0;
+  let paragraphs = 0;
+  // The first div that holds text; a second one means separate fields.
+  let div: Element | undefined;
+  const bearers = new Map<Node, Set<Node>>();
+  const add = (part: string) => (text = (text + part).replace(/\s+/g, " "));
+  // Marks `start` and its ancestors below root as carrying text. False once
+  // three siblings do.
+  const bear = (start: Node) => {
+    if (!strict) return true;
+    for (
+      let child = start, parent = start.parentNode;
+      child !== root && parent !== null;
+      child = parent, parent = parent.parentNode
+    ) {
+      const siblings = bearers.get(parent) ?? new Set<Node>();
+      bearers.set(parent, siblings);
+      if (siblings.has(child)) break;
+      if (siblings.add(child).size > 2) return false;
+    }
+    return true;
+  };
+  let node: Node | null = root.firstChild;
+  while (node !== null) {
+    if (++nodes > MAX_WALK_NODES || text.length > MAX_WALK_CHARS) {
+      if (strict) return "";
+      break;
+    }
+    let descend = false;
+    if (node instanceof Text) {
+      const parent = node.parentNode as Element;
+      // The block that holds this text, through inline elements.
+      let holder = parent;
+      while (holder !== root && PHRASING.test(holder.localName))
+        holder = holder.parentElement!;
+      if (
+        /\S/.test(node.data) &&
+        (!bear(parent) ||
+          (strict && holder.localName === "div" && (div ??= holder) !== holder))
+      )
+        return "";
+      add(node.data);
+    } else if (node instanceof Element) {
+      if (node.matches(EDITABLE)) return "";
+      if (
+        strict &&
+        node.matches(STRUCTURE) &&
+        !(root.localName === "summary" && node.matches(HEADING)) &&
+        (node.localName !== "p" || ++paragraphs > 1)
+      )
+        return "";
+      const name = node.matches(SKIPPED) ? null : imageName(node);
+      if (name) {
+        if (!bear(node)) return "";
+        add(` ${name} `);
+      } else if (name === undefined) {
+        if (!PHRASING.test(node.localName)) add(" ");
+        descend = true;
+      }
+    }
+    if (descend && node.firstChild !== null) node = node.firstChild;
+    else {
+      while (node !== root && node.nextSibling === null) {
+        node = node.parentNode!;
+        if (!PHRASING.test((node as Element).localName)) add(" ");
+      }
+      node = node === root ? null : node.nextSibling;
+    }
+  }
+  return text;
+}
+
+/** An option or tree item of an autocomplete or search popup, whose text usually repeats what the user typed. */
+const isSuggestion = (element: Element) =>
+  (element.localName === "option" ||
+    /^(option|treeitem)$/.test(roleOf(element))) &&
+  inSuggestionList(element);
+
+/** Whether `element` is, or is inside, a combobox, a datalist, or the popup a combobox or autocomplete field controls or owns. */
+function inSuggestionList(element: Element) {
+  if (element.closest("[role~=combobox], datalist") !== null) return true;
+  const root = rootOf(element);
+  return (
+    root !== null &&
+    Array.from(
+      root.querySelectorAll("[role~=combobox], [aria-autocomplete]"),
+    ).some((owner) =>
+      [...idRefs(owner, "aria-controls"), ...idRefs(owner, "aria-owns")].some(
+        (id) => root.getElementById(id)?.contains(element),
+      ),
+    )
+  );
+}
+
+function labelledBy(element: Element) {
+  const root = rootOf(element);
+  return root === null
+    ? null
+    : idRefs(element, "aria-labelledby")
+        .map((id) => {
+          const referenced = root.getElementById(id);
+          return referenced === null || inSuggestionList(referenced)
+            ? ""
+            : safeText(referenced, false);
+        })
+        .join(" ");
+}
+
+/** A control's own text, unless it is longer than a label: then it is content, not a name. */
+const normalize = (text: string) =>
+  text.replace(IGNORABLE, "").replace(/\s+/g, " ").trim();
+
+function contentText(element: Element) {
+  const text = safeText(element, true);
+  return Array.from(normalize(text)).length > MAX_LABEL_LENGTH ? "" : text;
+}
+
+/** Removes invisible characters, credentials, emails, URLs, and tokens, then caps the length. */
+function cleanLabel(raw: string, redact: Redactor) {
+  const chars = Array.from(
+    // Scrub again after collapsing, also for remembered values that differ
+    // in whitespace or invisible characters.
+    redact
+      .scrub(
+        normalize(redact.body(raw.replace(IGNORABLE, "")) ?? ""),
+        normalize,
+      )
+      .replace(URL_LIKE, "[url]")
+      .replace(EMAIL, "[email]")
+      .replace(JWT, `$1${REDACTED}`)
+      .replace(TOKEN_RUN, (run) => (/\d/.test(run) ? REDACTED : run)),
+  );
+  return chars.length > MAX_LABEL_LENGTH
+    ? `${chars
+        .slice(0, MAX_LABEL_LENGTH - 1)
+        .join("")
+        .trimEnd()}…`
+    : chars.join("");
+}
+
+function firstLabel(sources: (() => string | null)[], redact: Redactor) {
+  for (const source of sources) {
+    const label = cleanLabel(source() ?? "", redact);
+    if (label) return label;
+  }
+  return undefined;
+}
+
+/**
+ * A short name for the clicked control, from the first source that yields one:
+ * `aria-labelledby`, `aria-label`, an input caption or `alt`, `title` (controls
+ * only), then the text of a control named by its content. Containers, cards,
+ * and editable content never contribute their text. An autocomplete suggestion,
+ * or anything inside one, gets no label from any source; `data-copilotkit-action`
+ * names it instead. An icon without its own name takes the name of the control
+ * around it.
+ */
+function clickLabel(element: Element, redact: Redactor): string | undefined {
+  const option = element.closest("option, [role~=option], [role~=treeitem]");
+  if (option !== null && isSuggestion(option)) return undefined;
+  // aria-labelledby and aria-label apply to any other target.
+  const named = [
+    () => labelledBy(element),
+    () => element.getAttribute("aria-label"),
+  ];
+  if (
+    /^(img|svg)$/.test(element.localName) ||
+    /^(img|presentation|none)$/.test(roleOf(element))
+  ) {
+    const own = firstLabel(
+      [...named, () => imageName(element) ?? null],
+      redact,
+    );
+    if (own !== undefined) return own;
+    // parentElement is null at a shadow root, so the search stays inside it.
+    let control = element.parentElement;
+    while (control !== null && !isControl(control))
+      control = control.parentElement;
+    return control === null ? undefined : clickLabel(control, redact);
+  }
+  const control = isControl(element) && !holdsValue(element);
+  const captionInput =
+    element instanceof HTMLInputElement && CAPTION_INPUT.test(element.type);
+  return firstLabel(
+    [
+      ...named,
+      // The caption of a button-like input, or the `alt` of an image input.
+      () =>
+        captionInput
+          ? element.type === "image"
+            ? element.getAttribute("alt")
+            : element.value
+          : null,
+      () => (control || captionInput ? element.getAttribute("title") : null),
+      () => (control ? contentText(element) : null),
+    ],
+    redact,
+  );
+}
+
 /** Snapshot live control properties; HTML attributes do not reflect edited values. */
 export function readControlValue(element: Element, redact: Redactor) {
   if (element instanceof HTMLInputElement) {
@@ -96,7 +388,7 @@ export function readControlValue(element: Element, redact: Redactor) {
     };
   }
   if (element instanceof HTMLTextAreaElement) return { value: element.value };
-  if (element.matches('[contenteditable]:not([contenteditable="false"])')) {
+  if (element.matches(CONTENTEDITABLE)) {
     return { value: element.textContent ?? "" };
   }
   return {};
@@ -140,12 +432,25 @@ export function installClickCapture(params: {
   } = params;
   let origin: PointerOrigin | null = null;
 
+  // Input events share describeTarget; only clicks carry a label.
+  const describeClick = (el: Element) => {
+    const described = describeTarget(el, redact);
+    if (described === null) return null;
+    try {
+      const label = clickLabel(described.element, redact);
+      if (label !== undefined) described.target.label = label;
+    } catch {
+      // A failed label must not drop the click.
+    }
+    return described;
+  };
+
   const onPointerDown = (event: PointerEvent) => {
     try {
       if (event.isPrimary === false) return;
       const el = eventElement(event);
       origin = {
-        described: el === null ? null : describeTarget(el, redact),
+        described: el === null ? null : describeClick(el),
         x: event.clientX,
         y: event.clientY,
       };
@@ -167,7 +472,7 @@ export function installClickCapture(params: {
         isNear(pointerOrigin, event);
       const described = canRecover
         ? pointerOrigin.described
-        : describeTarget(el, redact);
+        : describeClick(el);
       if (described === null) return;
       const input = event.detail === 0 ? "keyboard" : "pointer";
       emit("click", {

package/src/collector.ts

@@ -27,6 +27,7 @@ interface Session {
   dropped: number;
   uninstalls: (() => void)[];
   timer: ReturnType<typeof setInterval> | undefined;
+  flushInputs?: () => void;
 }
 
 /**
@@ -79,6 +80,8 @@ export function createCollector(options: CollectorOptions) {
   const record = (name: string, value: Record<string, unknown>) => {
     const current = session;
     if (current === null) return;
+    if (name !== "input" && name !== "network") current.flushInputs?.();
+    if (session !== current) return;
     try {
       seq += 1;
       const event: LearningEvent = {
@@ -88,7 +91,7 @@ export function createCollector(options: CollectorOptions) {
         timestamp: Date.now(),
       };
       const kept = beforeSend === undefined ? event : beforeSend(event);
-      if (kept === null) return;
+      if (kept === null || session !== current) return;
       current.queue.push(kept);
       if (current.queue.length >= FLUSH_SIZE) flush();
     } catch {
@@ -146,10 +149,11 @@ export function createCollector(options: CollectorOptions) {
           redact,
         }),
       );
-    if (capture.inputs)
-      current.uninstalls.push(
-        installInputCapture({ emit: recordCurrent, redact }),
-      );
+    if (capture.inputs) {
+      const inputs = installInputCapture({ emit: recordCurrent, redact });
+      current.flushInputs = inputs.flush;
+      current.uninstalls.push(inputs);
+    }
     const onPageHide = () => flush({ beacon: true });
     window.addEventListener("pagehide", onPageHide);
     current.uninstalls.push(() =>
@@ -167,6 +171,8 @@ export function createCollector(options: CollectorOptions) {
   const stop = () => {
     const current = session;
     if (current === null) return;
+    current.flushInputs?.();
+    if (session !== current) return;
     clearInterval(current.timer);
     for (const uninstall of current.uninstalls) uninstall();
     flush();