@copilotkit/learning 0.0.1

package/src/collector.ts

@@ -0,0 +1,199 @@
+import { installClickCapture } from "./clicks";
+import { installInputCapture } from "./inputs";
+import { installNavigationCapture } from "./navigation";
+import { installNetworkCapture } from "./network";
+import { createRedactor } from "./redact";
+import { toRoute } from "./routes";
+import { BUILT_IN_EVENT_NAMES } from "./types";
+import type {
+  BuiltInEventName,
+  Collector,
+  CollectorOptions,
+  LearningBatch,
+  LearningEvent,
+  SinkSendOptions,
+  StartOptions,
+} from "./types";
+
+// ponytail: flushing at 50 also bounds the queue at 50; add a separate cap only if sends become async-buffered.
+const FLUSH_SIZE = 50;
+const FLUSH_INTERVAL_MS = 2000;
+const BUILT_IN = new Set<string>(BUILT_IN_EVENT_NAMES);
+
+interface Session {
+  trajectoryId: string;
+  learningContainerIds: string[] | undefined;
+  queue: LearningEvent[];
+  dropped: number;
+  uninstalls: (() => void)[];
+  timer: ReturnType<typeof setInterval> | undefined;
+}
+
+/**
+ * Creates a collector. Nothing is installed until `start()`: no listeners,
+ * no patches, no timers, no buffer. `stop()` removes all of them and sends
+ * queued events once, without retries.
+ *
+ * @example
+ * const collector = createCollector({ sink: httpSink("/api/learning-events") });
+ * collector.start({ trajectoryId: crypto.randomUUID() });
+ */
+export function createCollector(options: CollectorOptions) {
+  const { sink, routes, beforeSend, enrich } = options;
+  const capture = {
+    clicks: true,
+    navigation: true,
+    network: true,
+    inputs: true,
+    ...options.capture,
+  };
+  let session: Session | null = null;
+  let seq = 0;
+
+  const getRoute = () => toRoute(location.pathname, routes);
+
+  const flush = (sendOptions?: SinkSendOptions) => {
+    const current = session;
+    if (current === null) return;
+    if (current.queue.length === 0 && current.dropped === 0) return;
+    const events = current.queue.splice(0);
+    const dropped = current.dropped;
+    current.dropped = 0;
+    const batch: LearningBatch = {
+      trajectoryId: current.trajectoryId,
+      learningContainerIds: current.learningContainerIds,
+      events,
+      dropped,
+    };
+    const onFailure = () => {
+      // No retry: report the loss in the next batch instead.
+      if (session === current) current.dropped += events.length + dropped;
+    };
+    try {
+      Promise.resolve(sink(batch, sendOptions)).catch(onFailure);
+    } catch {
+      onFailure();
+    }
+  };
+
+  const record = (name: string, value: Record<string, unknown>) => {
+    const current = session;
+    if (current === null) return;
+    try {
+      seq += 1;
+      const event: LearningEvent = {
+        type: "CUSTOM",
+        name,
+        value: { ...value, seq },
+        timestamp: Date.now(),
+      };
+      const kept = beforeSend === undefined ? event : beforeSend(event);
+      if (kept === null) return;
+      current.queue.push(kept);
+      if (current.queue.length >= FLUSH_SIZE) flush();
+    } catch {
+      // Capture must never break the host app.
+    }
+  };
+
+  const start = ({ trajectoryId, learningContainerIds }: StartOptions) => {
+    if (session !== null) {
+      if (session.trajectoryId !== trajectoryId) {
+        console.warn(
+          `[@copilotkit/learning] Trajectory "${session.trajectoryId}" is active. Stop it before you start "${trajectoryId}".`,
+        );
+      }
+      return;
+    }
+    if (
+      typeof window === "undefined" ||
+      typeof window.addEventListener !== "function"
+    )
+      return;
+    const current: Session = {
+      trajectoryId,
+      learningContainerIds,
+      queue: [],
+      dropped: 0,
+      uninstalls: [],
+      timer: undefined,
+    };
+    session = current;
+    const recordCurrent: typeof record = (name, value) => {
+      if (session === current) record(name, value);
+    };
+    // One per session: fields seen and values typed are forgotten on stop().
+    const redact = createRedactor();
+    current.uninstalls.push(redact.watch());
+    const ignoreUrls = [
+      ...(options.ignoreUrls ?? []),
+      ...(sink.url === undefined ? [] : [sink.url]),
+    ];
+    if (capture.clicks)
+      current.uninstalls.push(
+        installClickCapture({ emit: recordCurrent, getRoute, enrich, redact }),
+      );
+    if (capture.navigation)
+      current.uninstalls.push(
+        installNavigationCapture({ emit: recordCurrent, routes, redact }),
+      );
+    if (capture.network)
+      current.uninstalls.push(
+        installNetworkCapture({
+          emit: recordCurrent,
+          routes,
+          ignoreUrls,
+          redact,
+        }),
+      );
+    if (capture.inputs)
+      current.uninstalls.push(
+        installInputCapture({ emit: recordCurrent, redact }),
+      );
+    const onPageHide = () => flush({ beacon: true });
+    window.addEventListener("pagehide", onPageHide);
+    current.uninstalls.push(() =>
+      window.removeEventListener("pagehide", onPageHide),
+    );
+    current.timer = setInterval(() => flush(), FLUSH_INTERVAL_MS);
+    record("page", {
+      route: getRoute(),
+      url: redact.url(location.href),
+      title: document.title,
+      referrer: redact.url(document.referrer),
+    });
+  };
+
+  const stop = () => {
+    const current = session;
+    if (current === null) return;
+    clearInterval(current.timer);
+    for (const uninstall of current.uninstalls) uninstall();
+    flush();
+    session = null;
+  };
+
+  const emit = (name: string, value: Record<string, unknown>) => {
+    if (BUILT_IN.has(name)) {
+      console.warn(
+        `[@copilotkit/learning] "${name}" is a built-in event name. Use your own name, such as "deal.approved".`,
+      );
+      return;
+    }
+    record(name, value);
+  };
+
+  const ɵemit = (name: BuiltInEventName, value: Record<string, unknown>) =>
+    record(name, value);
+
+  const collector: Collector = {
+    start,
+    stop,
+    emit,
+    ɵemit,
+    get trajectoryId() {
+      return session?.trajectoryId ?? null;
+    },
+  };
+  return collector;
+}

package/src/http-sink.ts

@@ -0,0 +1,38 @@
+import type { LearningBatch, LearningSink, SinkSendOptions } from "./types";
+
+/**
+ * A sink that POSTs each batch as JSON to `url`.
+ *
+ * Uses `fetch` with `keepalive`. When the page is going away it uses
+ * `navigator.sendBeacon`, which cannot send custom headers. There is no retry:
+ * a failed send rejects, and the collector counts those events as dropped.
+ *
+ * @example createCollector({ sink: httpSink("/api/learning-events") })
+ */
+export function httpSink(
+  url: string,
+  init: { headers?: Record<string, string> } = {},
+) {
+  const send = async (batch: LearningBatch, options: SinkSendOptions = {}) => {
+    const body = JSON.stringify(batch);
+    const canBeacon =
+      options.beacon === true && typeof navigator.sendBeacon === "function";
+    if (canBeacon) {
+      const queued = navigator.sendBeacon(
+        url,
+        new Blob([body], { type: "application/json" }),
+      );
+      if (!queued) throw new Error("sendBeacon refused the batch");
+      return;
+    }
+    const response = await fetch(url, {
+      method: "POST",
+      keepalive: true,
+      headers: { "content-type": "application/json", ...init.headers },
+      body,
+    });
+    if (!response.ok) throw new Error(`Sink responded with ${response.status}`);
+  };
+  const sink: LearningSink = Object.assign(send, { url });
+  return sink;
+}

package/src/index.ts

@@ -0,0 +1,30 @@
+export { createCollector } from "./collector";
+export { httpSink } from "./http-sink";
+export { toOriginAndRoute, toRoute } from "./routes";
+export { BUILT_IN_EVENT_NAMES } from "./types";
+export {
+  createTrajectoryCollector,
+  MAX_TRAJECTORY_EVENT_BYTES,
+} from "./trajectory-collector";
+export type {
+  JsonValue,
+  TrajectoryEvent,
+  StartResult,
+  ConnectionGrant,
+  TrajectoryError,
+  TrajectoryCaptureOptions,
+  TrajectoryCollector,
+} from "./trajectory-types";
+export type {
+  BuiltInEventName,
+  CaptureOptions,
+  ClickTarget,
+  Collector,
+  CollectorOptions,
+  EnrichFn,
+  LearningBatch,
+  LearningEvent,
+  LearningSink,
+  SinkSendOptions,
+  StartOptions,
+} from "./types";

package/src/inputs.ts

@@ -0,0 +1,42 @@
+import { describeTarget, readControlValue } from "./clicks";
+import { createRedactor } from "./redact";
+import type { Redactor } from "./redact";
+import type { Emit } from "./types";
+
+/** Records user edits without changing the control or interfering with its listeners. */
+export function installInputCapture(params: {
+  emit: Emit;
+  isTrusted?: (event: Event) => boolean;
+  redact?: Redactor;
+}) {
+  const {
+    emit,
+    isTrusted = (event: Event) => event.isTrusted,
+    redact = createRedactor(),
+  } = params;
+  const onInput = (event: Event) => {
+    try {
+      if (!isTrusted(event)) return;
+      const [element] = event.composedPath();
+      if (!(element instanceof Element)) return;
+      const described = describeTarget(element, redact);
+      if (described === null) return;
+      const value = readControlValue(element, redact);
+      if (!("value" in value)) return;
+      emit("input", {
+        eventType: event.type,
+        target: { ...described.target, ...value },
+        url: redact.url(location.href),
+        route: location.pathname,
+      });
+    } catch {
+      // Capture must never break an edit in the host app.
+    }
+  };
+  window.addEventListener("input", onInput, { capture: true, passive: true });
+  window.addEventListener("change", onInput, { capture: true, passive: true });
+  return () => {
+    window.removeEventListener("input", onInput, { capture: true });
+    window.removeEventListener("change", onInput, { capture: true });
+  };
+}

package/src/navigation.ts

@@ -0,0 +1,123 @@
+import { createRedactor } from "./redact";
+import type { Redactor } from "./redact";
+import type { Emit } from "./types";
+
+type NavigationType = "push" | "replace" | "traverse" | "reload";
+type HistoryMethod = "pushState" | "replaceState";
+
+/** The part of the Navigation API this module uses (not in TypeScript's DOM lib yet). */
+interface NavigationTarget {
+  addEventListener(
+    type: "currententrychange",
+    listener: (event: Event) => void,
+  ): void;
+  removeEventListener(
+    type: "currententrychange",
+    listener: (event: Event) => void,
+  ): void;
+}
+
+function isNavigationTarget(value: unknown): value is NavigationTarget {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    "addEventListener" in value &&
+    "removeEventListener" in value
+  );
+}
+
+function getNavigation() {
+  const navigation: unknown = Reflect.get(window, "navigation");
+  return isNavigationTarget(navigation) ? navigation : null;
+}
+
+function toNavigationType(value: unknown): NavigationType {
+  switch (value) {
+    case "replace":
+    case "traverse":
+    case "reload":
+      return value;
+    default:
+      return "push";
+  }
+}
+
+/**
+ * Emits `navigation` when the full URL changes in a single-page app.
+ * Uses the Navigation API when it exists. Otherwise patches `pushState` and
+ * `replaceState` on the prototype and on the instance, because some routers
+ * (Next.js) wrap the instance method.
+ */
+export function installNavigationCapture(params: {
+  emit: Emit;
+  routes?: string[];
+  redact?: Redactor;
+}) {
+  const { emit, redact = createRedactor() } = params;
+  let current = location.href;
+
+  const check = (navigationType: NavigationType) => {
+    try {
+      const next = location.href;
+      if (next === current) return;
+      emit("navigation", {
+        from: redact.url(current),
+        to: redact.url(next),
+        navigationType,
+      });
+      current = next;
+    } catch {
+      // Capture must never break navigation.
+    }
+  };
+
+  const navigation = getNavigation();
+  if (navigation !== null) {
+    const onEntryChange = (event: Event) =>
+      check(toNavigationType(Reflect.get(event, "navigationType")));
+    navigation.addEventListener("currententrychange", onEntryChange);
+    return () =>
+      navigation.removeEventListener("currententrychange", onEntryChange);
+  }
+
+  return installHistoryFallback(check);
+}
+
+function installHistoryFallback(
+  check: (navigationType: NavigationType) => void,
+) {
+  let active = true;
+  const restores: (() => void)[] = [];
+
+  const patch = (target: History, method: HistoryMethod) => {
+    const original = target[method];
+    const wrapper: History[HistoryMethod] = function (this: History, ...args) {
+      const result = original.apply(this, args);
+      if (active) check(method === "pushState" ? "push" : "replace");
+      return result;
+    };
+    target[method] = wrapper;
+    restores.push(() => {
+      // Restore only if no one wrapped it after us; otherwise our wrapper stays as a pass-through.
+      if (target[method] === wrapper) target[method] = original;
+    });
+  };
+
+  const methods: HistoryMethod[] = ["pushState", "replaceState"];
+  for (const method of methods) {
+    patch(History.prototype, method);
+    if (Object.prototype.hasOwnProperty.call(history, method))
+      patch(history, method);
+  }
+
+  const onPopState = () => check("traverse");
+  window.addEventListener("popstate", onPopState);
+  window.addEventListener("hashchange", onPopState);
+
+  return () => {
+    active = false;
+    window.removeEventListener("popstate", onPopState);
+    window.removeEventListener("hashchange", onPopState);
+    for (const restore of restores) restore();
+  };
+}

package/src/network-body.ts

@@ -0,0 +1,237 @@
+import { TEXT_LIMIT } from "./redact";
+import type { Redactor } from "./redact";
+
+/** A bounded observation of a body; never the stream returned to the app. */
+export interface BodySnapshot {
+  status:
+    | "complete"
+    | "empty"
+    | "truncated"
+    | "unavailable"
+    | "interrupted"
+    | "timeout";
+  text?: string;
+  encoding?: "utf-8" | "base64";
+  truncated?: true;
+  reason?: string;
+}
+
+const BODY_BYTES = 4096;
+const READ_TIMEOUT_MS = 1000;
+const encoder = new TextEncoder();
+const redactionFailed = (): BodySnapshot => ({
+  status: "unavailable",
+  reason: "redaction-failed",
+});
+type RedactText = (text: string) => string | undefined;
+
+/** Redacts up to TEXT_LIMIT bytes of text first, then cuts the result to BODY_BYTES. */
+function snapshot(
+  bytes: Uint8Array,
+  status: BodySnapshot["status"],
+  contentType: string,
+  redact: RedactText,
+  reason?: string,
+): BodySnapshot {
+  let text: string | undefined;
+  if (!isBinary(contentType))
+    try {
+      // Declared text decodes lossily so it is always redacted; untyped bytes
+      // that are not UTF-8 are treated as binary.
+      text = new TextDecoder("utf-8", { fatal: !contentType }).decode(bytes, {
+        stream: status !== "complete",
+      });
+    } catch {
+      /* Not UTF-8: keep the bytes as base64. */
+    }
+  if (text !== undefined) {
+    text = redact(text);
+    if (text === undefined) return redactionFailed();
+    // PostgreSQL jsonb rejects NUL in strings; base64 retains the redacted bytes.
+    if (text.includes("\0")) {
+      bytes = encoder.encode(text);
+      text = undefined;
+    }
+  }
+  const binary = text === undefined;
+  // Binary bodies are not inspected; only their first bytes are kept.
+  text ??= btoa(String.fromCharCode(...bytes.subarray(0, BODY_BYTES)));
+  const result: BodySnapshot = {
+    status: status === "complete" && bytes.length === 0 ? "empty" : status,
+    text,
+    encoding: binary ? "base64" : "utf-8",
+    ...(reason ? { reason } : {}),
+  };
+  if (encoder.encode(JSON.stringify(result)).length <= BODY_BYTES)
+    return result;
+  result.status = status === "complete" ? "truncated" : status;
+  result.truncated = true;
+  // Bound the serialized field too: quotes/control characters expand in JSON.
+  let low = 0;
+  let high = text.length;
+  while (low < high) {
+    const middle = Math.ceil((low + high) / 2);
+    result.text = text.slice(0, middle);
+    if (encoder.encode(JSON.stringify(result)).length <= BODY_BYTES)
+      low = middle;
+    else high = middle - 1;
+  }
+  if (binary)
+    low -= low % 4; // Keep complete base64 groups.
+  else if (low > 0 && /[\uD800-\uDBFF]/.test(text[low - 1]!)) low--;
+  result.text = text.slice(0, low);
+  return result;
+}
+
+function snapshotText(
+  value: string,
+  redact: RedactText,
+  status: BodySnapshot["status"] = "complete",
+  reason?: string,
+): BodySnapshot {
+  return snapshot(
+    encoder.encode(value.slice(0, TEXT_LIMIT)),
+    value.length > TEXT_LIMIT && status === "complete" ? "truncated" : status,
+    "text/plain",
+    redact,
+    reason,
+  );
+}
+
+function isBinary(contentType: string) {
+  return (
+    contentType !== "" &&
+    !/(^text\/|json|xml|javascript|graphql|x-www-form-urlencoded|multipart\/form-data)/i.test(
+      contentType,
+    )
+  );
+}
+
+/** Cancel only cloned/synthetic bodies. Cancelling a tee may wait for the app's branch. */
+export function createBodyCapture(redact: Redactor) {
+  const cancellations = new Set<() => void>();
+  let active = true;
+  const read = (
+    stream: ReadableStream<Uint8Array> | null,
+    contentType = "",
+  ): Promise<BodySnapshot> => {
+    if (!stream) return Promise.resolve({ status: "empty" });
+    if (!active)
+      return Promise.resolve({
+        status: "unavailable",
+        reason: "capture-stopped",
+      });
+    let reader: ReadableStreamDefaultReader<Uint8Array>;
+    try {
+      reader = stream.getReader();
+    } catch {
+      return Promise.resolve({ status: "unavailable", reason: "body-locked" });
+    }
+    return new Promise((resolve) => {
+      const bytes = new Uint8Array(TEXT_LIMIT);
+      let length = 0;
+      let finished = false;
+      const finish = (status: BodySnapshot["status"], reason?: string) => {
+        if (finished) return;
+        finished = true;
+        clearTimeout(timer);
+        cancellations.delete(cancel);
+        resolve(
+          snapshot(
+            bytes.subarray(0, length),
+            status,
+            contentType,
+            redact.body,
+            reason,
+          ),
+        );
+        // Do not await cancellation: a cloned fetch stream can share a live SSE source.
+        void reader.cancel().catch(() => {});
+      };
+      const cancel = () => finish("interrupted", "capture-stopped");
+      const timer = setTimeout(
+        () => finish("timeout", "body-read-timeout"),
+        READ_TIMEOUT_MS,
+      );
+      cancellations.add(cancel);
+      void (async () => {
+        try {
+          for (;;) {
+            const chunk = await reader.read();
+            if (finished) break;
+            if (chunk.done) {
+              finish("complete");
+              break;
+            }
+            const kept = chunk.value.subarray(0, TEXT_LIMIT - length);
+            bytes.set(kept, length);
+            length += kept.length;
+            if (length === TEXT_LIMIT) {
+              finish("truncated");
+              break;
+            }
+          }
+        } catch {
+          finish("interrupted", "body-read-failed");
+        } finally {
+          try {
+            reader.releaseLock();
+          } catch {
+            /* A cancelled read may still be settling. */
+          }
+        }
+      })();
+    });
+  };
+  const body = (
+    value: BodyInit | null | undefined,
+    contentType = "",
+  ): Promise<BodySnapshot> => {
+    try {
+      if (value == null) return Promise.resolve({ status: "empty" });
+      if (value instanceof URLSearchParams) value = value.toString();
+      if (typeof value === "string")
+        return Promise.resolve(snapshotText(value, redact.body));
+      if (
+        typeof ReadableStream !== "undefined" &&
+        value instanceof ReadableStream
+      ) {
+        // A caller-owned init.body cannot be teed without locking/replacing it.
+        return Promise.resolve({
+          status: "unavailable",
+          reason: "caller-owned-stream",
+        });
+      }
+      if (value instanceof FormData) {
+        const form = new FormData();
+        value.forEach((field, key) =>
+          form.append(
+            key,
+            typeof field === "string" ? redact.field(key, field) : field,
+          ),
+        );
+        value = form;
+      }
+      const response = new Response(value);
+      return read(
+        response.body,
+        contentType || response.headers.get("content-type") || "",
+      );
+    } catch {
+      return Promise.resolve({
+        status: "unavailable",
+        reason: "body-not-readable",
+      });
+    }
+  };
+  return {
+    read,
+    body,
+    text: (value: string, status?: BodySnapshot["status"], reason?: string) =>
+      snapshotText(value, redact.body, status, reason),
+    stop() {
+      active = false;
+      for (const cancel of cancellations) cancel();
+    },
+  };
+}