@copilotkit/learning 0.0.1

package/src/network.ts

@@ -0,0 +1,394 @@
+import { createBodyCapture } from "./network-body";
+import type { BodySnapshot } from "./network-body";
+import { createRedactor, isCredentialKey } from "./redact";
+import type { Redactor } from "./redact";
+import { REDACTED } from "./types";
+import type { Emit } from "./types";
+
+type Outcome = "ok" | "error" | "aborted";
+interface RequestParts {
+  url: string;
+  method: string;
+  headers: Record<string, string>;
+}
+interface ResponseParts {
+  url?: string;
+  headers: Record<string, string>;
+  body: Promise<BodySnapshot>;
+}
+const emptyBody = () => Promise.resolve<BodySnapshot>({ status: "empty" });
+const unavailableBody = (reason: string) =>
+  Promise.resolve<BodySnapshot>({ status: "unavailable", reason });
+const toAbsolute = (url: string) => new URL(url, location.href).href;
+// Credentials carry no product signal and must never leave the browser, even
+// with full capture. The header name stays, so the event still shows it was sent.
+// Names the credential-key matcher misses; it covers `authorization`, `cookie`,
+// `x-api-key`, `x-csrf-token`, and similar.
+const CREDENTIAL_HEADERS = new Set([
+  "x-amz-security-token",
+  "private-token",
+  "x-token",
+]);
+const headerValue = (name: string, value: string) =>
+  CREDENTIAL_HEADERS.has(name.toLowerCase()) || isCredentialKey(name)
+    ? REDACTED
+    : value;
+const headerValues = (headers: Headers) =>
+  Object.fromEntries(
+    Array.from(headers.entries(), ([name, value]) => [
+      name,
+      headerValue(name, value),
+    ]),
+  );
+
+// Checks the name, not instanceof: errors can come from another realm.
+function isAbortError(error: unknown) {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "name" in error &&
+    error.name === "AbortError"
+  );
+}
+
+/** Records browser-visible request/response data without consuming the app's bodies. */
+export function installNetworkCapture(params: {
+  emit: Emit;
+  routes?: string[];
+  ignoreUrls: (string | RegExp)[];
+  redact?: Redactor;
+}) {
+  let active = true;
+  const { redact = createRedactor() } = params;
+  const bodies = createBodyCapture(redact);
+  const ignoreUrls = params.ignoreUrls.map((pattern) =>
+    typeof pattern === "string"
+      ? toAbsolute(pattern)
+      : new RegExp(pattern.source, pattern.flags),
+  );
+  const isIgnored = (url: string) =>
+    ignoreUrls.some((pattern) => {
+      if (typeof pattern === "string") return url.startsWith(pattern);
+      pattern.lastIndex = 0;
+      return pattern.test(url);
+    });
+
+  const record: RecordFn = (
+    transport,
+    request,
+    requestBody,
+    startedAt,
+    status,
+    outcome,
+    response,
+  ) => {
+    const durationMs = Math.round(performance.now() - startedAt);
+    const completedAt = Date.now();
+    void Promise.all([
+      requestBody,
+      response?.body ?? unavailableBody("no-response"),
+    ])
+      .then(([sent, received]) => {
+        if (!active) return;
+        const href = redact.url(request.url);
+        const url = new URL(href);
+        params.emit("network", {
+          transport,
+          method: request.method,
+          url: href,
+          origin: url.origin,
+          route: url.pathname,
+          status,
+          durationMs,
+          completedAt,
+          outcome,
+          request: { headers: request.headers, body: sent },
+          response: {
+            ...(response?.url ? { url: redact.url(response.url) } : {}),
+            headers: response?.headers ?? {},
+            body: received,
+          },
+        });
+      })
+      .catch(() => {
+        /* Capture must never affect the host request. */
+      });
+  };
+
+  const originalFetch = globalThis.fetch;
+  const wrappedFetch: typeof fetch = (input, init) => {
+    if (!active) return originalFetch.call(globalThis, input, init);
+    let request: RequestParts;
+    let ignored: boolean;
+    try {
+      const source =
+        typeof Request !== "undefined" && input instanceof Request
+          ? input
+          : undefined;
+      request = {
+        url: toAbsolute(source ? source.url : String(input)),
+        method: (init?.method ?? source?.method ?? "GET").toUpperCase(),
+        headers: headerValues(new Headers(init?.headers ?? source?.headers)),
+      };
+      ignored = isIgnored(request.url);
+    } catch {
+      return originalFetch.call(globalThis, input, init);
+    }
+    if (ignored) return originalFetch.call(globalThis, input, init);
+    const startedAt = performance.now();
+    let requestBody: Promise<BodySnapshot>;
+    try {
+      requestBody =
+        init?.body != null
+          ? bodies.body(init.body, request.headers["content-type"])
+          : typeof Request !== "undefined" && input instanceof Request
+            ? bodies.read(input.clone().body, request.headers["content-type"])
+            : emptyBody();
+    } catch {
+      requestBody = unavailableBody("request-clone-failed");
+    }
+    const failed = (error: unknown) =>
+      record(
+        "fetch",
+        request,
+        requestBody,
+        startedAt,
+        null,
+        isAbortError(error) ? "aborted" : "error",
+      );
+    let promise: Promise<Response>;
+    try {
+      promise = originalFetch.call(globalThis, input, init);
+    } catch (error) {
+      failed(error);
+      throw error;
+    }
+    void promise
+      .then((response) => {
+        if (!active) return;
+        let responseBody: Promise<BodySnapshot>;
+        try {
+          responseBody =
+            response.type === "opaque" || response.type === "opaqueredirect"
+              ? unavailableBody("opaque-response")
+              : bodies.read(
+                  response.clone().body,
+                  response.headers.get("content-type") ?? "",
+                );
+        } catch {
+          responseBody = unavailableBody("response-clone-failed");
+        }
+        record(
+          "fetch",
+          request,
+          requestBody,
+          startedAt,
+          response.status,
+          "ok",
+          {
+            url: response.url,
+            headers: headerValues(response.headers),
+            body: responseBody,
+          },
+        );
+      }, failed)
+      .catch(() => {});
+    // Return the identical native Promise, Response, and rejection to the caller.
+    return promise;
+  };
+  if (typeof originalFetch === "function") globalThis.fetch = wrappedFetch;
+  const uninstallXhr = patchXhr(isIgnored, record, bodies);
+  return () => {
+    active = false;
+    bodies.stop();
+    if (globalThis.fetch === wrappedFetch) globalThis.fetch = originalFetch;
+    uninstallXhr();
+  };
+}
+
+type RecordFn = (
+  transport: "fetch" | "xhr",
+  request: RequestParts,
+  requestBody: Promise<BodySnapshot>,
+  startedAt: number,
+  status: number | null,
+  outcome: Outcome,
+  response?: ResponseParts,
+) => void;
+
+function patchXhr(
+  isIgnored: (url: string) => boolean,
+  record: RecordFn,
+  bodies: ReturnType<typeof createBodyCapture>,
+) {
+  if (typeof XMLHttpRequest !== "function") return () => {};
+  const proto = XMLHttpRequest.prototype;
+  const originalOpen = proto.open;
+  const originalSend = proto.send;
+  const originalSetHeader = proto.setRequestHeader;
+  const requests = new WeakMap<XMLHttpRequest, RequestParts>();
+  const listeners = new Set<() => void>();
+  // open() on an in-flight XHR ends it without `loadend`; its listeners must not
+  // report the next request with the old request's details.
+  const inFlight = new WeakMap<XMLHttpRequest, () => void>();
+  let active = true;
+  const wrappedOpen = function (
+    this: XMLHttpRequest,
+    method: string,
+    url: string | URL,
+    ...rest: unknown[]
+  ) {
+    inFlight.get(this)?.();
+    const result = Reflect.apply(originalOpen, this, [method, url, ...rest]);
+    try {
+      requests.set(this, {
+        url: toAbsolute(String(url)),
+        method: method.toUpperCase(),
+        headers: {},
+      });
+    } catch {
+      requests.delete(this);
+    }
+    return result;
+  };
+  const wrappedSetHeader = function (
+    this: XMLHttpRequest,
+    name: string,
+    value: string,
+  ) {
+    const result = originalSetHeader.call(this, name, value);
+    const request = requests.get(this);
+    if (active && request) {
+      try {
+        const headers = new Headers(request.headers);
+        headers.append(name, value);
+        request.headers = headerValues(headers);
+      } catch {
+        /* Preserve successful native calls even if observation fails. */
+      }
+    }
+    return result;
+  };
+  const wrappedSend = function (
+    this: XMLHttpRequest,
+    body?: Document | XMLHttpRequestBodyInit | null,
+  ) {
+    const request = requests.get(this);
+    if (!active || !request || isIgnored(request.url))
+      return originalSend.call(this, body);
+    const startedAt = performance.now();
+    let requestBody: Promise<BodySnapshot>;
+    try {
+      requestBody =
+        typeof Document !== "undefined" && body instanceof Document
+          ? Promise.resolve(
+              bodies.text(new XMLSerializer().serializeToString(body)),
+            )
+          : bodies.body(
+              body as BodyInit | null | undefined,
+              request.headers["content-type"],
+            );
+    } catch {
+      requestBody = unavailableBody("request-body-unavailable");
+    }
+    let aborted = false;
+    const onAbort = () => {
+      aborted = true;
+    };
+    const cleanup = () => {
+      this.removeEventListener("abort", onAbort);
+      this.removeEventListener("loadend", onEnd);
+      listeners.delete(cleanup);
+      if (inFlight.get(this) === cleanup) inFlight.delete(this);
+    };
+    const onEnd = () => {
+      cleanup();
+      if (!active) return;
+      const outcome = aborted ? "aborted" : this.status === 0 ? "error" : "ok";
+      const headers: Record<string, string> = {};
+      try {
+        for (const line of this.getAllResponseHeaders().split(/\r?\n/)) {
+          const split = line.indexOf(":");
+          if (split > 0) {
+            const name = line.slice(0, split).toLowerCase();
+            headers[name] = headerValue(name, line.slice(split + 1).trim());
+          }
+        }
+      } catch {
+        /* Cross-origin or failed requests can hide response headers. */
+      }
+      let responseBody: Promise<BodySnapshot>;
+      try {
+        if (this.responseType === "" || this.responseType === "text") {
+          responseBody = Promise.resolve(
+            bodies.text(
+              this.responseText,
+              outcome === "ok" ? "complete" : "interrupted",
+              outcome === "ok" ? undefined : `request-${outcome}`,
+            ),
+          );
+        } else if (this.responseType === "json")
+          responseBody = Promise.resolve(
+            bodies.text(JSON.stringify(this.response)),
+          );
+        else if (this.responseType === "document")
+          responseBody = this.responseXML
+            ? Promise.resolve(
+                bodies.text(
+                  new XMLSerializer().serializeToString(this.responseXML),
+                ),
+              )
+            : unavailableBody("empty-document");
+        else
+          responseBody = bodies.body(
+            this.response as BodyInit,
+            headers["content-type"] ?? "application/octet-stream",
+          );
+      } catch {
+        responseBody = unavailableBody("response-body-unavailable");
+      }
+      record(
+        "xhr",
+        request,
+        requestBody,
+        startedAt,
+        outcome === "ok" ? this.status : null,
+        outcome,
+        {
+          url: this.responseURL,
+          headers,
+          body: responseBody,
+        },
+      );
+    };
+    this.addEventListener("abort", onAbort);
+    this.addEventListener("loadend", onEnd);
+    listeners.add(cleanup);
+    inFlight.set(this, cleanup);
+    try {
+      return originalSend.call(this, body);
+    } catch (error) {
+      cleanup();
+      record(
+        "xhr",
+        request,
+        requestBody,
+        startedAt,
+        null,
+        isAbortError(error) ? "aborted" : "error",
+      );
+      throw error;
+    }
+  };
+  proto.open = wrappedOpen;
+  proto.send = wrappedSend;
+  proto.setRequestHeader = wrappedSetHeader;
+  return () => {
+    active = false;
+    for (const cleanup of listeners) cleanup();
+    if (proto.open === wrappedOpen) proto.open = originalOpen;
+    if (proto.send === wrappedSend) proto.send = originalSend;
+    if (proto.setRequestHeader === wrappedSetHeader)
+      proto.setRequestHeader = originalSetHeader;
+  };
+}

package/src/redact.ts

@@ -0,0 +1,257 @@
+import { REDACTED } from "./types";
+
+// Passwords and unambiguous credentials never leave the browser, even with full
+// capture. Keys are split into lowercase words (`accessToken`, `access_token`,
+// `Access-Token` all become "access token"). These match anywhere in the joined words:
+const CREDENTIAL =
+  /password|passwd|passphrase|passcode|credential|api(key|token)|privatekey|subscriptionkey|(access|refresh|auth|session)token|clientsecret/;
+// These match only as whole words (`secret` but not `secretary`, `id token` but
+// not `invalid token`), as the last word (`Set-Cookie` but not `cookieConsent`),
+// or as the entire key.
+const CREDENTIAL_WORD =
+  / (secret|id ?token)s? | (authorization|cookies?|jwt|bearer ?token|[cx]srf ?token) $|^ (pwd|pass|token|otp) $/;
+// Metadata about a credential is not the credential: `passwordExpiresAt`, `credentialId`.
+const METADATA =
+  / (ids?|at|in|expires?|expiry|length|count|enabled|required|policy|hint|type) $/;
+const PASSWORD_AUTOCOMPLETE = /(current|new)-password|one-time-code/;
+// A JSON string, possibly cut off, with the `:` that makes it a key.
+const TOKEN = /"((?:\\[\s\S]|[^"\\])*)("?)(\s*:\s*)?/g;
+// A lone trailing backslash belongs to a cut-off string.
+const STRING = /"(?:\\[\s\S]?|[^"\\])*"?/y;
+// `key=value` and `key: value` outside JSON strings: forms, query strings,
+// GraphQL, YAML. A `:` needs a space or end after it, so `host:port` is no pair.
+// The key length bound keeps scanning long unbroken text linear.
+// An auth scheme (`Bearer x`) belongs to the value.
+const PAIR =
+  /([\w$.%+[\]-]{1,64})(\s*(?:=|:(?=\s|$))\s*)((?:(?:basic|bearer|digest|token) +)?[^\s&,;)}\]]*)/gi;
+// `scheme://user:pass@` anywhere in text; up to the last `@` before the host.
+// The scheme length bound keeps scanning long words linear.
+const USERINFO = /([a-z][\w+.-]{0,31}:\/\/)[^\s/?#"]*@/gi;
+/** Text beyond this is neither redacted nor kept; snapshots keep 4 KiB of the result. */
+export const TEXT_LIMIT = 16 * 1024;
+const REMEMBERED_VALUES = 20;
+
+export const isCredentialKey = (key: string) => {
+  const words = ` ${key
+    .replace(/%([\da-f]{2})/gi, (_, hex: string) =>
+      String.fromCharCode(parseInt(hex, 16)),
+    )
+    .replace(/([a-z\d])(?=[A-Z])/g, "$1 ")
+    .toLowerCase()
+    .split(/[^a-z\d]+/)
+    .filter(Boolean)
+    .join(" ")} `;
+  return (
+    !METADATA.test(words) &&
+    (CREDENTIAL.test(words.replace(/ /g, "")) || CREDENTIAL_WORD.test(words))
+  );
+};
+
+const decode = (body: string) => {
+  try {
+    return JSON.parse(`"${body}"`) as string;
+  } catch {
+    return body;
+  }
+};
+
+/** The end of the JSON value at `start`: a string, a balanced object or array, or a bare literal. */
+const endOfValue = (text: string, start: number) => {
+  let depth = 0;
+  for (let i = start; i < text.length; i++) {
+    const char = text[i]!;
+    if (char === '"') {
+      STRING.lastIndex = i;
+      STRING.test(text);
+      i = STRING.lastIndex - 1;
+    } else if (char === "{" || char === "[") {
+      depth++;
+      continue;
+    } else if (char === "}" || char === "]") {
+      if (--depth < 0) return i;
+    } else {
+      if (!depth && (char === "," || /\s/.test(char))) return i;
+      continue;
+    }
+    if (!depth) return i + 1;
+  }
+  return text.length;
+};
+
+/**
+ * Redacts credential values in JSON (valid or cut off) and in key/value text,
+ * and recurses into string values that hold JSON, forms, or URLs. Unchanged
+ * parts keep their exact original text.
+ */
+const redactText = (text: string, depth = 0): string => {
+  const token = new RegExp(TOKEN);
+  let pending = false;
+  const pairs = (part: string) => {
+    const segment = part.replace(USERINFO, "$1");
+    return segment.replace(
+      PAIR,
+      (pair, key: string, separator: string, value: string, offset: number) => {
+        if (!isCredentialKey(key)) return pair;
+        // The value may continue in a following string: `password:"x"`.
+        if (offset + pair.length === segment.length) {
+          pending = true;
+          if (!value) return pair;
+        }
+        return key + separator + REDACTED;
+      },
+    );
+  };
+  let out = "";
+  let last = 0;
+  let match: RegExpExecArray | null;
+  while ((match = token.exec(text))) {
+    const [raw, body, closed, colon] = match;
+    pending = false;
+    out += pairs(text.slice(last, match.index));
+    let end = token.lastIndex;
+    let replacement = raw;
+    const value = body!.includes("\\") ? decode(body!) : body!;
+    if (colon) {
+      if (isCredentialKey(value)) {
+        const valueEnd = endOfValue(text, end);
+        // Booleans and null say whether a credential exists, not what it is.
+        if (!/^(true|false|null|)$/.test(text.slice(end, valueEnd))) {
+          replacement += `"${REDACTED}"`;
+          end = valueEnd;
+        }
+      }
+    } else if (pending) replacement = `"${REDACTED}"`;
+    else if (depth < 4 && /[=:"]/.test(value)) {
+      const redacted = redactText(value, depth + 1);
+      if (redacted !== value)
+        replacement = JSON.stringify(redacted).slice(
+          0,
+          closed ? undefined : -1,
+        );
+    }
+    out += replacement;
+    last = token.lastIndex = end;
+  }
+  return out + pairs(text.slice(last));
+};
+
+/** Per-session redaction state: password-like fields and the values typed into them. */
+export function createRedactor() {
+  const seen = new WeakSet<Element>();
+  // Latest value per field, so each keystroke replaces the previous prefix.
+  const remembered = new Map<Element, string>();
+  let observer: MutationObserver | undefined;
+
+  const toggled = (records: MutationRecord[]) => {
+    for (const record of records)
+      if (record.oldValue?.toLowerCase() === "password")
+        seen.add(record.target as Element);
+  };
+
+  /** Replaces remembered values, raw and in their JSON and URL encodings. */
+  const scrub = (text: string) => {
+    for (const value of [...remembered.values()].sort(
+      (a, b) => b.length - a.length,
+    ))
+      for (const form of [
+        value,
+        JSON.stringify(value).slice(1, -1),
+        encodeURIComponent(value),
+        new URLSearchParams({ v: value }).toString().slice(2),
+      ])
+        text = /^\d+$/.test(form)
+          ? // One-time codes must not match inside longer numbers.
+            text.replace(RegExp(`(?<!\\d)${form}(?!\\d)`, "g"), REDACTED)
+          : text.split(form).join(REDACTED);
+    return text;
+  };
+
+  /** True for current, former (show-password toggles), and autocomplete password fields. */
+  const isPassword = (element: Element) => {
+    if (observer) toggled(observer.takeRecords());
+    if (
+      !(element instanceof HTMLInputElement) ||
+      !(
+        element.type === "password" ||
+        seen.has(element) ||
+        isCredentialKey(element.name) ||
+        PASSWORD_AUTOCOMPLETE.test(element.getAttribute("autocomplete") ?? "")
+      )
+    )
+      return false;
+    seen.add(element);
+    if (element.value.length >= 4) {
+      remembered.delete(element);
+      remembered.set(element, element.value);
+      if (remembered.size > REMEMBERED_VALUES)
+        remembered.delete(remembered.keys().next().value!);
+    }
+    return true;
+  };
+
+  return {
+    isPassword,
+    /**
+     * Tracks password fields for the session: existing ones, `type` toggles away
+     * from "password", and typed values even when input capture is off.
+     */
+    watch() {
+      // Each part is optional: a partial DOM (tests, embedded webviews, hardened
+      // pages) must not stop capture from starting or break the host app.
+      try {
+        for (const input of document.querySelectorAll("input[type=password]"))
+          seen.add(input);
+      } catch {
+        // Fields are still caught on first sight by isPassword.
+      }
+      let current: MutationObserver | undefined;
+      try {
+        current = new MutationObserver(toggled);
+        current.observe(document, {
+          subtree: true,
+          attributeFilter: ["type"],
+          attributeOldValue: true,
+        });
+        observer = current;
+      } catch {
+        current = undefined;
+      }
+      const remember = (event: Event) => {
+        try {
+          const [target] = event.composedPath();
+          if (target instanceof Element) isPassword(target);
+        } catch {
+          // Capture must never break an edit in the host app.
+        }
+      };
+      for (const type of ["input", "change"])
+        window.addEventListener(type, remember, true);
+      return () => {
+        current?.disconnect();
+        if (current !== undefined && observer === current) observer = undefined;
+        for (const type of ["input", "change"])
+          window.removeEventListener(type, remember, true);
+      };
+    },
+    field: (key: string, value: string) =>
+      isCredentialKey(key) ? REDACTED : scrub(redactText(value)),
+    /** Returns the redacted body text, or undefined if redaction failed. */
+    body(text: string) {
+      try {
+        return scrub(redactText(text.slice(0, TEXT_LIMIT)));
+      } catch {
+        return undefined;
+      }
+    },
+    /** Drops userinfo and redacts credential values; also applied to element attributes. */
+    url(value: string) {
+      try {
+        return scrub(value.replace(/[^#]+/g, (part) => redactText(part)));
+      } catch {
+        return REDACTED;
+      }
+    },
+  };
+}
+
+export type Redactor = ReturnType<typeof createRedactor>;

package/src/routes.ts

@@ -0,0 +1,14 @@
+/** Returns the original pathname. Capture no longer masks or categorizes routes. */
+export function toRoute(pathname: string, _templates: string[] = []) {
+  return pathname;
+}
+
+/** Resolves a URL without removing path segments, query parameters, or fragments. */
+export function toOriginAndRoute(
+  url: string,
+  _templates: string[] = [],
+  base: string = globalThis.location?.href ?? "http://localhost/",
+) {
+  const parsed = new URL(url, base);
+  return { url: parsed.href, origin: parsed.origin, route: parsed.pathname };
+}