@agent-native/core 0.133.2 → 0.134.0

package/corpus/core/src/tracking/providers.ts

@@ -13,6 +13,7 @@
  * automatically by the core-routes plugin).
  */
 
+import { reshapeTrackedExceptionProperties } from "./posthog-exception.js";
 import { registerTrackingProvider } from "./registry.js";
 import type { TrackingProvider, TrackingEvent } from "./types.js";
 
@@ -162,27 +163,61 @@ function isPostHogAiObservabilityEvent(eventName: string): boolean {
   return eventName.startsWith("$ai_");
 }
 
-function createPostHogProvider(apiKey: string, host: string): TrackingProvider {
+/**
+ * `$ai_*` and `$exception` are ingested through PostHog's dedicated endpoint
+ * rather than `/capture/`, and `$exception` additionally has to carry
+ * `$exception_list` — the framework's own `captureException()` emits camelCase
+ * fields that PostHog would otherwise render as an empty, ungroupable issue.
+ */
+function createPostHogProvider(
+  apiKey: string,
+  host: string,
+  errorTracking: boolean,
+): TrackingProvider {
+  const sendToEventsEndpoint = (
+    event: TrackingEvent,
+    properties: Record<string, unknown> | undefined,
+    distinctId: string,
+  ): void => {
+    enqueue(
+      `${host}/i/v0/e/`,
+      JSON.stringify({
+        api_key: apiKey,
+        event: event.name,
+        properties: {
+          distinct_id: distinctId,
+          ...properties,
+          timestamp: event.timestamp,
+        },
+      }),
+    );
+  };
+
   return {
     name: "posthog",
     track(event: TrackingEvent) {
       const distinctId = event.userId || "anonymous";
       if (isPostHogAiObservabilityEvent(event.name)) {
-        enqueue(
-          `${host}/i/v0/e/`,
-          JSON.stringify({
-            api_key: apiKey,
-            event: event.name,
-            properties: {
-              distinct_id: distinctId,
-              ...event.properties,
-              timestamp: event.timestamp,
-            },
-          }),
-        );
+        sendToEventsEndpoint(event, event.properties, distinctId);
         return;
       }
 
+      if (event.name === "$exception") {
+        // `POSTHOG_ERROR_TRACKING=false` keeps product analytics flowing while
+        // another backend owns crashes. Drop rather than downgrade to
+        // `/capture/`: a malformed exception is what this branch exists to
+        // prevent.
+        if (!errorTracking) return;
+        const reshaped = reshapeTrackedExceptionProperties(event.properties);
+        if (reshaped) {
+          sendToEventsEndpoint(event, reshaped, distinctId);
+          return;
+        }
+        // No recognizable exception fields. Fall through to `/capture/` so the
+        // event is still recorded as-is rather than becoming an issue with
+        // nothing in it.
+      }
+
       enqueue(
         `${host}/capture/`,
         JSON.stringify({
@@ -213,6 +248,42 @@ function createPostHogProvider(apiKey: string, host: string): TrackingProvider {
   };
 }
 
+/**
+ * Send one event to PostHog only, bypassing the provider fan-out.
+ *
+ * For payloads whose *shape* is PostHog-specific and whose *content* other
+ * backends must not receive. `track()` broadcasts to every configured provider,
+ * so a PostHog-only integration (e.g. enabling a survey id) would otherwise
+ * start exporting that integration's content — including user-authored text —
+ * to Mixpanel, Amplitude, webhooks, and Agent Native Analytics as a side
+ * effect nobody opted into.
+ *
+ * Returns `false` when PostHog is not configured, so callers can tell "not
+ * sent" from "sent".
+ */
+export function sendPostHogEvent(
+  name: string,
+  properties: Record<string, unknown>,
+  distinctId: string,
+): boolean {
+  const apiKey = process.env.POSTHOG_API_KEY;
+  if (!apiKey) return false;
+  const host = (process.env.POSTHOG_HOST || POSTHOG_DEFAULT_HOST).replace(
+    /\/+$/,
+    "",
+  );
+  enqueue(
+    `${host}/capture/`,
+    JSON.stringify({
+      api_key: apiKey,
+      event: name,
+      distinct_id: distinctId,
+      properties: { ...properties, timestamp: new Date().toISOString() },
+    }),
+  );
+  return true;
+}
+
 // ─── Mixpanel ──────────────────────────────────────────────────────────────
 
 function createMixpanelProvider(token: string): TrackingProvider {
@@ -383,7 +454,13 @@ export function registerBuiltinProviders(): void {
       /\/+$/,
       "",
     );
-    registerTrackingProvider(createPostHogProvider(posthogKey, host));
+    registerTrackingProvider(
+      createPostHogProvider(
+        posthogKey,
+        host,
+        process.env.POSTHOG_ERROR_TRACKING?.trim().toLowerCase() !== "false",
+      ),
+    );
   }
 
   const mixpanelToken = process.env.MIXPANEL_TOKEN;

package/corpus/core/src/tracking/redaction.ts

@@ -0,0 +1,105 @@
+/**
+ * Bounding and redaction helpers shared by every exception emitter.
+ *
+ * Kept free of `process.env` and Node built-ins so the browser bundle can use
+ * the same rules — an exception shaped one way on the server and another way
+ * in the client is how a leak ships in only one of them.
+ */
+
+export const MAX_MESSAGE_LENGTH = 1000;
+export const MAX_STACK_LENGTH = 8000;
+export const MAX_TAGS = 30;
+export const MAX_EXTRA_KEYS = 30;
+export const MAX_EXTRA_VALUE_LENGTH = 1000;
+
+const SECRET_RE = /\b(?:bearer|basic)\s+[^\s]+/gi;
+
+export const SECRET_KEY_RE =
+  /(?:authorization|cookie|set[-_]?cookie|token|secret|password|passwd|pwd|api[-_]?key|apikey|credential)/i;
+
+export function redact(value: string): string {
+  return value
+    .replace(SECRET_RE, (match) => `${match.split(/\s+/, 1)[0]} <redacted>`)
+    .replace(
+      /([A-Za-z0-9_$.-]*(?:authorization|cookie|token|secret|password|passwd|pwd|api[-_]?key|apikey|credential)[A-Za-z0-9_$.-]*\s*[:=]\s*)([^\s,;}]+)/gi,
+      "$1<redacted>",
+    );
+}
+
+export function boundedText(value: unknown, max: number): string {
+  const text = typeof value === "string" ? value : String(value ?? "");
+  const safe = redact(text);
+  return safe.length > max ? safe.slice(0, max) : safe;
+}
+
+export function safeValue(value: unknown, depth = 2): unknown {
+  if (
+    value == null ||
+    typeof value === "boolean" ||
+    typeof value === "number"
+  ) {
+    return value;
+  }
+  if (typeof value === "string")
+    return boundedText(value, MAX_EXTRA_VALUE_LENGTH);
+  if (depth <= 0) return boundedText(value, MAX_EXTRA_VALUE_LENGTH);
+  if (Array.isArray(value)) {
+    return value.slice(0, 20).map((item) => safeValue(item, depth - 1));
+  }
+  if (typeof value === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [key, child] of Object.entries(value)) {
+      if (Object.keys(out).length >= MAX_EXTRA_KEYS) break;
+      const safeKey = boundedText(key, 100);
+      out[safeKey] = SECRET_KEY_RE.test(safeKey)
+        ? "<redacted>"
+        : safeValue(child, depth - 1);
+    }
+    return out;
+  }
+  return boundedText(value, MAX_EXTRA_VALUE_LENGTH);
+}
+
+export function safeTags(
+  tags: Record<string, string | undefined> | undefined,
+): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const [key, value] of Object.entries(tags ?? {})) {
+    if (Object.keys(out).length >= MAX_TAGS) break;
+    // Skip, don't stop: callers build tag objects with optional entries
+    // (`{ ...tags, route, method, userAgent }`), so breaking on the first
+    // undefined dropped every tag after it depending on key order.
+    if (value == null) continue;
+    const safeKey = boundedText(key, 100);
+    out[safeKey] = SECRET_KEY_RE.test(safeKey)
+      ? "<redacted>"
+      : boundedText(value, 200);
+  }
+  return out;
+}
+
+export interface ExceptionParts {
+  type: string;
+  message: string;
+  stack?: string;
+}
+
+/** Split an unknown thrown value into bounded, redacted type/message/stack. */
+export function exceptionParts(error: unknown): ExceptionParts {
+  if (error instanceof Error) {
+    return {
+      type: boundedText(error.name || "Error", 200),
+      message: boundedText(
+        error.message || error.name || "Error",
+        MAX_MESSAGE_LENGTH,
+      ),
+      ...(error.stack
+        ? { stack: boundedText(error.stack, MAX_STACK_LENGTH) }
+        : {}),
+    };
+  }
+  return {
+    type: "Error",
+    message: boundedText(error, MAX_MESSAGE_LENGTH),
+  };
+}

package/corpus/templates/content/actions/_batch-utils.ts

@@ -1,3 +1,5 @@
+import { getDialect, type Dialect } from "@agent-native/core/db";
+
 export function chunks<T>(items: T[], size: number): T[][] {
   if (items.length === 0) return [];
   const out: T[][] = [];
@@ -6,3 +8,34 @@ export function chunks<T>(items: T[], size: number): T[][] {
   }
   return out;
 }
+
+export function bulkChunkSizeForColumnCount(
+  columnCount: number,
+  dialect: Dialect = getDialect(),
+) {
+  // D1 rejects statements with more than 100 bound params, so derive every
+  // bulk chunk from the statement's column count instead of a fixed row count.
+  const budget = dialect === "d1" ? 90 : 900;
+  return Math.max(1, Math.floor(budget / Math.max(1, columnCount)));
+}
+
+export async function processWithConcurrency<T>(
+  items: readonly T[],
+  concurrency: number,
+  worker: (item: T) => Promise<void>,
+) {
+  let nextIndex = 0;
+  const workerCount = Math.min(
+    items.length,
+    Math.max(1, Math.floor(concurrency)),
+  );
+  await Promise.all(
+    Array.from({ length: workerCount }, async () => {
+      while (nextIndex < items.length) {
+        const item = items[nextIndex];
+        nextIndex += 1;
+        if (item !== undefined) await worker(item);
+      }
+    }),
+  );
+}

package/corpus/templates/content/actions/_builder-cms-read-client.ts

@@ -163,6 +163,7 @@ type BuilderMcpToolResult = {
 const BUILDER_CMS_DEFAULT_READ_LIMIT = 500;
 const BUILDER_CMS_MAX_READ_LIMIT = 10_000;
 const BUILDER_CMS_PAGE_SIZE = 100;
+const BUILDER_CMS_PROJECTED_READ_CONCURRENCY = 8;
 const BUILDER_CMS_READ_RETRIES = 2;
 const BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS = [
   "id",
@@ -192,6 +193,10 @@ const BUILDER_CMS_HEAVY_BODY_FIELD_PATHS = [
 const BUILDER_CMS_FIELD_PATH_PATTERN =
   /^[A-Za-z0-9_$-]+(?:\.[A-Za-z0-9_$-]+)*$/;
 
+type BuilderCmsContentPageResult =
+  | { pageLimit: number; pageEntries: BuilderCmsSourceEntry[] }
+  | { pageLimit: number; error: string };
+
 function normalizeBuilderCmsListFieldPath(fieldPath: string) {
   const trimmed = fieldPath.trim();
   if (!trimmed || !BUILDER_CMS_FIELD_PATH_PATTERN.test(trimmed)) return null;
@@ -218,6 +223,36 @@ function normalizeBuilderCmsListFieldPath(fieldPath: string) {
   return normalized;
 }
 
+function preserveProjectedBuilderFieldAbsence(
+  read: BuilderCmsReadResult,
+  fieldPaths: readonly string[] | undefined,
+  rawData: boolean | undefined,
+): BuilderCmsReadResult {
+  if (read.state !== "live" || rawData === true || !fieldPaths?.length) {
+    return read;
+  }
+  const projectedFieldPaths = [
+    ...new Set(
+      fieldPaths
+        .map(normalizeBuilderCmsListFieldPath)
+        .filter((fieldPath): fieldPath is string => fieldPath !== null),
+    ),
+  ];
+  if (projectedFieldPaths.length === 0) return read;
+  return {
+    ...read,
+    entries: read.entries.map((entry) => {
+      const sourceValues = { ...entry.sourceValues };
+      for (const fieldPath of projectedFieldPaths) {
+        if (!Object.prototype.hasOwnProperty.call(sourceValues, fieldPath)) {
+          sourceValues[fieldPath] = null;
+        }
+      }
+      return { ...entry, sourceValues };
+    }),
+  };
+}
+
 export function builderCmsListEntryFields(fieldPaths: readonly string[] = []) {
   const fields = new Map<string, string>();
   for (const fieldPath of [
@@ -752,6 +787,7 @@ async function initializeBuilderMcp(args: {
 async function readBuilderCmsContentEntriesViaMcp(args: {
   model: string;
   fieldPaths?: readonly string[];
+  includeBodies?: boolean;
   rawData?: boolean;
   limit?: number;
   maxPages?: number;
@@ -775,7 +811,9 @@ async function readBuilderCmsContentEntriesViaMcp(args: {
   const contentEntries: BuilderCmsSourceEntry[] = [];
   const fields = args.rawData
     ? undefined
-    : builderCmsListEntryFields(args.fieldPaths);
+    : args.includeBodies
+      ? `${builderCmsListEntryFields(args.fieldPaths)},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}`
+      : builderCmsListEntryFields(args.fieldPaths);
   const seenContentIds = new Set<string>();
   let pagesRead = 0;
   let hasMore = false;
@@ -941,6 +979,8 @@ async function readBuilderCmsContentEntriesViaMcp(args: {
 async function readBuilderCmsContentEntriesViaContentApi(args: {
   model: string;
   fieldPaths?: readonly string[];
+  includeBodies?: boolean;
+  allowCached?: boolean;
   rawData?: boolean;
   limit?: number;
   maxPages?: number;
@@ -959,14 +999,21 @@ async function readBuilderCmsContentEntriesViaContentApi(args: {
   // Fidelity reads intentionally do neither: Builder's raw data object is the
   // clone contract, including unresolved references and every nested field.
   url.searchParams.set("enrich", args.rawData === true ? "false" : "true");
-  url.searchParams.set("noCache", "true");
-  url.searchParams.set(
-    "cachebust",
-    args.rawData === true ? String(Date.now()) : "true",
-  );
+  if (args.allowCached !== true || args.rawData === true) {
+    url.searchParams.set("noCache", "true");
+    url.searchParams.set(
+      "cachebust",
+      args.rawData === true ? String(Date.now()) : "true",
+    );
+  }
   url.searchParams.set("includeUnpublished", "true");
   if (args.rawData !== true) {
-    url.searchParams.set("fields", builderCmsListEntryFields(args.fieldPaths));
+    url.searchParams.set(
+      "fields",
+      args.includeBodies
+        ? `${builderCmsListEntryFields(args.fieldPaths)},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}`
+        : builderCmsListEntryFields(args.fieldPaths),
+    );
   }
 
   const limit = readLimit(args.limit);
@@ -978,73 +1025,103 @@ async function readBuilderCmsContentEntriesViaContentApi(args: {
   const seenIds = new Set<string>();
   let pagesRead = 0;
   let hasMore = false;
-  for (
-    let offset = startOffset;
-    entries.length < limit;
-    offset += BUILDER_CMS_PAGE_SIZE
-  ) {
-    const pageUrl = new URL(url);
-    const pageLimit = readPageLimit(limit - entries.length);
-    pageUrl.searchParams.set("limit", String(pageLimit));
-    pageUrl.searchParams.set("offset", String(offset));
+  const concurrency =
+    args.rawData === true ? 1 : BUILDER_CMS_PROJECTED_READ_CONCURRENCY;
+  const errorResult = (message: string): BuilderCmsReadResult => ({
+    state: "error",
+    entries: [],
+    fetchedAt,
+    message,
+    progress: {
+      requestedLimit: limit,
+      pageSize: BUILDER_CMS_PAGE_SIZE,
+      startOffset,
+      nextOffset: startOffset + entries.length,
+      fetchedEntryCount: startOffset + entries.length,
+      hasMore,
+      partial: Boolean(args.maxPages) && hasMore,
+      readMode: "builder-api",
+    },
+  });
 
-    let response: Response;
-    try {
-      response = await fetchBuilderContentPage({
-        fetchImpl: args.fetchImpl,
-        url: pageUrl,
-        privateKey: args.privateKey,
-      });
-    } catch (error) {
-      return {
-        state: "error",
-        entries: [],
-        fetchedAt,
-        message:
-          error instanceof Error
-            ? `Builder CMS read failed: ${error.message}`
-            : "Builder CMS read failed.",
-        progress: {
-          requestedLimit: limit,
-          pageSize: BUILDER_CMS_PAGE_SIZE,
-          startOffset,
-          nextOffset: startOffset + entries.length,
-          fetchedEntryCount: startOffset + entries.length,
-          hasMore,
-          partial: Boolean(args.maxPages) && hasMore,
-          readMode: "builder-api",
+  while (entries.length < limit) {
+    const pagesRemaining = args.maxPages
+      ? args.maxPages - pagesRead
+      : Number.POSITIVE_INFINITY;
+    if (pagesRemaining <= 0) break;
+    const pageCount = Math.min(
+      pagesRead === 0 ? 1 : concurrency,
+      pagesRemaining,
+      Math.ceil((limit - entries.length) / BUILDER_CMS_PAGE_SIZE),
+    );
+    const pageRequests = Array.from({ length: pageCount }, (_, index) => {
+      const pageUrl = new URL(url);
+      const pageLimit = readPageLimit(
+        limit - entries.length - index * BUILDER_CMS_PAGE_SIZE,
+      );
+      pageUrl.searchParams.set("limit", String(pageLimit));
+      pageUrl.searchParams.set(
+        "offset",
+        String(startOffset + (pagesRead + index) * BUILDER_CMS_PAGE_SIZE),
+      );
+      return { pageLimit, pageUrl };
+    });
+    const pageResults = await Promise.all(
+      pageRequests.map(
+        async ({
+          pageLimit,
+          pageUrl,
+        }): Promise<BuilderCmsContentPageResult> => {
+          try {
+            const response = await fetchBuilderContentPage({
+              fetchImpl: args.fetchImpl,
+              url: pageUrl,
+              privateKey: args.privateKey,
+            });
+            if (!response.ok) {
+              return {
+                pageLimit,
+                error: `Builder CMS read failed with HTTP ${response.status}.`,
+              };
+            }
+            const json = (await response.json()) as unknown;
+            const pageEntries = entryArrayFromResponse(json)
+              .map((entry) => normalizeBuilderCmsApiEntry(entry, args.model))
+              .filter((entry): entry is BuilderCmsSourceEntry =>
+                Boolean(entry),
+              );
+            return { pageLimit, pageEntries };
+          } catch (error) {
+            return {
+              pageLimit,
+              error:
+                error instanceof Error
+                  ? `Builder CMS read failed: ${error.message}`
+                  : "Builder CMS read failed.",
+            };
+          }
         },
-      };
-    }
+      ),
+    );
 
-    if (!response.ok) {
-      return {
-        state: "error",
-        entries: [],
-        fetchedAt,
-        message: `Builder CMS read failed with HTTP ${response.status}.`,
-        progress: {
-          requestedLimit: limit,
-          pageSize: BUILDER_CMS_PAGE_SIZE,
-          startOffset,
-          nextOffset: startOffset + entries.length,
-          fetchedEntryCount: startOffset + entries.length,
-          hasMore,
-          partial: Boolean(args.maxPages) && hasMore,
-          readMode: "builder-api",
-        },
-      };
+    let stoppedOnShortPage = false;
+    for (const pageResult of pageResults) {
+      if ("error" in pageResult) return errorResult(pageResult.error);
+      const appended = appendUniqueBuilderEntries(
+        entries,
+        seenIds,
+        pageResult.pageEntries,
+      );
+      pagesRead += 1;
+      hasMore =
+        pageResult.pageEntries.length >= pageResult.pageLimit && appended > 0;
+      if (args.maxPages && pagesRead >= args.maxPages) break;
+      if (!hasMore) {
+        stoppedOnShortPage = true;
+        break;
+      }
     }
-
-    const json = (await response.json()) as unknown;
-    const pageEntries = entryArrayFromResponse(json)
-      .map((entry) => normalizeBuilderCmsApiEntry(entry, args.model))
-      .filter((entry): entry is BuilderCmsSourceEntry => Boolean(entry));
-    const appended = appendUniqueBuilderEntries(entries, seenIds, pageEntries);
-    pagesRead += 1;
-    hasMore = pageEntries.length >= pageLimit && appended > 0;
-    if (args.maxPages && pagesRead >= args.maxPages) break;
-    if (!hasMore) break;
+    if (stoppedOnShortPage || !hasMore) break;
   }
 
   return {
@@ -1231,6 +1308,8 @@ export async function readBuilderCmsModelFields(args: {
 export async function readBuilderCmsContentEntries(args: {
   model: string;
   fieldPaths?: readonly string[];
+  includeBodies?: boolean;
+  allowCached?: boolean;
   rawData?: boolean;
   requirePrivateKey?: boolean;
   limit?: number;
@@ -1265,6 +1344,8 @@ export async function readBuilderCmsContentEntries(args: {
     const contentApiRead = await readBuilderCmsContentEntriesViaContentApi({
       model: args.model,
       fieldPaths: args.fieldPaths,
+      includeBodies: args.includeBodies,
+      allowCached: args.allowCached,
       rawData: args.rawData,
       limit: args.limit,
       maxPages: args.maxPages,
@@ -1273,22 +1354,33 @@ export async function readBuilderCmsContentEntries(args: {
       publicKey,
       privateKey: privateKey ?? undefined,
     });
-    if (contentApiRead.state === "live") return contentApiRead;
+    if (contentApiRead.state === "live") {
+      return preserveProjectedBuilderFieldAbsence(
+        contentApiRead,
+        args.fieldPaths,
+        args.rawData,
+      );
+    }
     if (!privateKey) return contentApiRead;
   }
 
   if (privateKey) {
     try {
-      return await readBuilderCmsContentEntriesViaMcp({
-        model: args.model,
-        fieldPaths: args.fieldPaths,
-        rawData: args.rawData,
-        limit: args.limit,
-        maxPages: args.maxPages,
-        offset: args.offset,
-        fetchImpl,
-        privateKey,
-      });
+      return preserveProjectedBuilderFieldAbsence(
+        await readBuilderCmsContentEntriesViaMcp({
+          model: args.model,
+          fieldPaths: args.fieldPaths,
+          includeBodies: args.includeBodies,
+          rawData: args.rawData,
+          limit: args.limit,
+          maxPages: args.maxPages,
+          offset: args.offset,
+          fetchImpl,
+          privateKey,
+        }),
+        args.fieldPaths,
+        args.rawData,
+      );
     } catch (error) {
       return {
         state: "error",