@agent-native/core 0.189.0 → 0.190.0-nightly-20260925215540

package/corpus/templates/analytics/server/lib/analytics-query-catalog.ts

@@ -1,7 +1,7 @@
 import {
-  getAllSettings,
   getUserSetting,
   listOrgSettings,
+  listSettingsByPrefix,
 } from "@agent-native/core/settings";
 
 import { dashboardCatalogEntries } from "./dashboard-catalog";
@@ -19,6 +19,9 @@ import {
 const DATA_DICTIONARY_KEY_PREFIX = "data-dict-";
 const MAX_QUERY_LENGTH = 12_000;
 const MAX_CATALOG_DASHBOARD_HYDRATION = 24;
+// ponytail: scan the 200 most recently updated summaries; a search-backed index is the upgrade path if larger workspaces need older references.
+const MAX_CATALOG_DASHBOARD_SUMMARIES = 200;
+const MAX_CATALOG_DICTIONARY_ENTRIES = 200;
 const RETIRED_CATALOG_STATES = new Set([
   "deprecated",
   "obsolete",
@@ -118,11 +121,29 @@ export type AnalyticsQueryCatalogCandidate =
       columnsUsed?: string;
       queryTemplate?: string;
       knownGotchas?: string;
+      commonQuestions?: string;
+      cuts?: string;
+      joinPattern?: string;
+      updateFrequency?: string;
+      dataLag?: string;
+      dependencies?: string;
+      validDateRange?: string;
+      owner?: string;
       approved?: boolean;
       aiGenerated?: boolean;
       sourceUrl?: string;
     };
 
+export type AnalyticsQueryCatalogSearchResult = {
+  candidates: AnalyticsQueryCatalogCandidate[];
+  searchedDashboardCount: number;
+  dashboardSearchTruncated: boolean;
+  dashboardSearchStatus: "available" | "unavailable";
+  searchedDictionaryEntryCount: number;
+  dictionarySearchTruncated: boolean;
+  dictionarySearchStatus: "available" | "partial" | "unavailable";
+};
+
 function text(value: unknown): string {
   return typeof value === "string" ? value.trim() : "";
 }
@@ -452,7 +473,7 @@ function dictionaryCandidates(
   entries: DictionaryEntry[],
   search: string,
 ): AnalyticsQueryCatalogCandidate[] {
-  return entries.flatMap((entry) => {
+  const candidates = entries.flatMap((entry) => {
     if (isRetiredCatalogReference(entry)) return [];
     const { score, matchedTerms } = matchScore(search, [
       { value: entry.metric, weight: 28 },
@@ -493,6 +514,24 @@ function dictionaryCandidates(
         ...(text(entry.knownGotchas)
           ? { knownGotchas: text(entry.knownGotchas) }
           : {}),
+        ...(text(entry.commonQuestions)
+          ? { commonQuestions: text(entry.commonQuestions) }
+          : {}),
+        ...(text(entry.cuts) ? { cuts: text(entry.cuts) } : {}),
+        ...(text(entry.joinPattern)
+          ? { joinPattern: text(entry.joinPattern) }
+          : {}),
+        ...(text(entry.updateFrequency)
+          ? { updateFrequency: text(entry.updateFrequency) }
+          : {}),
+        ...(text(entry.dataLag) ? { dataLag: text(entry.dataLag) } : {}),
+        ...(text(entry.dependencies)
+          ? { dependencies: text(entry.dependencies) }
+          : {}),
+        ...(text(entry.validDateRange)
+          ? { validDateRange: text(entry.validDateRange) }
+          : {}),
+        ...(text(entry.owner) ? { owner: text(entry.owner) } : {}),
         ...(typeof entry.approved === "boolean"
           ? { approved: entry.approved }
           : {}),
@@ -503,6 +542,19 @@ function dictionaryCandidates(
       },
     ];
   });
+  const strongestHumanOrApprovedScore = candidates.reduce(
+    (strongest, candidate) =>
+      candidate.approved === true || candidate.aiGenerated !== true
+        ? Math.max(strongest, candidate.score)
+        : strongest,
+    0,
+  );
+  return candidates.filter(
+    (candidate) =>
+      candidate.aiGenerated !== true ||
+      candidate.approved === true ||
+      candidate.score > strongestHumanOrApprovedScore,
+  );
 }
 
 function candidateIsRunnable(
@@ -525,7 +577,9 @@ function candidateDedupeKey(candidate: AnalyticsQueryCatalogCandidate): string {
     .toLowerCase()}`;
 }
 
-function candidateTrustTier(candidate: AnalyticsQueryCatalogCandidate): number {
+export function candidateTrustTier(
+  candidate: AnalyticsQueryCatalogCandidate,
+): number {
   if (candidate.kind !== "dashboard-panel") return 0;
   if (candidate.dashboardCertified) return 2;
   return candidate.favorite ? 1 : 0;
@@ -591,7 +645,12 @@ export function rankAnalyticsQueryCatalog(args: {
 async function listDictionaryEntries(args: {
   email: string;
   orgId: string | null;
-}): Promise<DictionaryEntry[]> {
+}): Promise<{
+  entries: DictionaryEntry[];
+  searchedEntryCount: number;
+  truncated: boolean;
+  status: "available" | "partial" | "unavailable";
+}> {
   const entries: DictionaryEntry[] = [];
   const seen = new Set<string>();
   const collect = (raw: unknown) => {
@@ -603,20 +662,76 @@ async function listDictionaryEntries(args: {
     entries.push(entry);
   };
 
-  if (args.orgId) {
-    const orgEntries = await listOrgSettings(
-      args.orgId,
-      DATA_DICTIONARY_KEY_PREFIX,
+  const userPrefix = `u:${args.email}:${DATA_DICTIONARY_KEY_PREFIX}`;
+  const [orgResult, userResult] = await Promise.allSettled([
+    args.orgId
+      ? listOrgSettings(args.orgId, DATA_DICTIONARY_KEY_PREFIX, {
+          limit: MAX_CATALOG_DICTIONARY_ENTRIES + 1,
+        })
+      : Promise.resolve(null),
+    listSettingsByPrefix(userPrefix, {
+      limit: MAX_CATALOG_DICTIONARY_ENTRIES + 1,
+    }),
+  ]);
+  if (orgResult.status === "fulfilled" && orgResult.value) {
+    const orgEntries = Object.entries(orgResult.value);
+    for (const [, value] of orgEntries.slice(
+      0,
+      MAX_CATALOG_DICTIONARY_ENTRIES,
+    )) {
+      collect(value);
+    }
+  } else if (orgResult.status === "rejected") {
+    console.warn(
+      "[analytics-query-catalog] Organization dictionary lookup failed:",
+      orgResult.reason,
     );
-    for (const value of Object.values(orgEntries)) collect(value);
   }
-
-  const userPrefix = `u:${args.email}:${DATA_DICTIONARY_KEY_PREFIX}`;
-  const all = await getAllSettings();
-  for (const [key, value] of Object.entries(all)) {
-    if (key.startsWith(userPrefix)) collect(value);
+  if (userResult.status === "fulfilled") {
+    for (const { value } of userResult.value.slice(
+      0,
+      MAX_CATALOG_DICTIONARY_ENTRIES,
+    )) {
+      collect(value);
+    }
+  } else {
+    console.warn(
+      "[analytics-query-catalog] User dictionary lookup failed:",
+      userResult.reason,
+    );
   }
-  return entries;
+  const orgCount =
+    orgResult.status === "fulfilled"
+      ? Object.keys(orgResult.value ?? {}).length
+      : 0;
+  const userCount =
+    userResult.status === "fulfilled" ? userResult.value.length : 0;
+  const scopedResults = [
+    ...(args.orgId ? [orgResult.status === "fulfilled"] : []),
+    userResult.status === "fulfilled",
+  ];
+  const availableScopes = scopedResults.filter(Boolean).length;
+  const scopeCount = args.orgId ? 2 : 1;
+  return {
+    entries,
+    searchedEntryCount: entries.length,
+    truncated:
+      orgCount > MAX_CATALOG_DICTIONARY_ENTRIES ||
+      userCount > MAX_CATALOG_DICTIONARY_ENTRIES,
+    status:
+      availableScopes === scopeCount
+        ? "available"
+        : availableScopes > 0
+          ? "partial"
+          : "unavailable",
+  };
+}
+
+function warnCatalogReadFailure(source: string, error: unknown): void {
+  console.warn(
+    `[analytics-query-catalog] ${source} lookup failed; continuing with available references:`,
+    error,
+  );
 }
 
 function savedDashboardInput(
@@ -640,8 +755,10 @@ export async function searchAnalyticsQueryCatalog(args: {
   email: string;
   orgId: string | null;
   limit: number;
-}): Promise<AnalyticsQueryCatalogCandidate[]> {
-  const [savedSummariesResult, dictionaryEntriesResult, favoriteIdsResult] =
+  signal?: AbortSignal;
+}): Promise<AnalyticsQueryCatalogSearchResult> {
+  args.signal?.throwIfAborted();
+  const [summariesResult, dictionaryResult, favoritesResult] =
     await Promise.allSettled([
       listDashboardSummaries(
         { email: args.email, orgId: args.orgId },
@@ -650,38 +767,83 @@ export async function searchAnalyticsQueryCatalog(args: {
           archived: "active",
           hidden: "visible",
           includeCatalogMetadata: true,
+          limit: MAX_CATALOG_DASHBOARD_SUMMARIES + 1,
         },
       ),
       listDictionaryEntries({ email: args.email, orgId: args.orgId }),
       listFavoriteDashboardIds(args.email),
     ]);
-  const favoriteIds =
-    favoriteIdsResult.status === "fulfilled"
-      ? favoriteIdsResult.value
-      : new Set<string>();
+  args.signal?.throwIfAborted();
   const savedSummaries =
-    savedSummariesResult.status === "fulfilled"
-      ? savedSummariesResult.value
-      : [];
+    summariesResult.status === "fulfilled" ? summariesResult.value : [];
+  const searchedSummaries = savedSummaries.slice(
+    0,
+    MAX_CATALOG_DASHBOARD_SUMMARIES,
+  );
+  const dashboardSearchTruncated =
+    savedSummaries.length > MAX_CATALOG_DASHBOARD_SUMMARIES;
+  if (dashboardSearchTruncated) {
+    console.warn("[analytics] Dashboard reference search truncated.", {
+      searchedDashboardCount: searchedSummaries.length,
+      dashboardSearchTruncated,
+    });
+  }
   const dictionaryEntries =
-    dictionaryEntriesResult.status === "fulfilled"
-      ? dictionaryEntriesResult.value
+    dictionaryResult.status === "fulfilled"
+      ? dictionaryResult.value.entries
       : [];
+  const searchedDictionaryEntryCount =
+    dictionaryResult.status === "fulfilled"
+      ? dictionaryResult.value.searchedEntryCount
+      : 0;
+  const dictionarySearchTruncated =
+    dictionaryResult.status === "fulfilled" && dictionaryResult.value.truncated;
+  let dashboardSearchStatus: "available" | "unavailable" =
+    summariesResult.status === "fulfilled" ? "available" : "unavailable";
+  const dictionarySearchStatus =
+    dictionaryResult.status === "fulfilled"
+      ? dictionaryResult.value.status
+      : "unavailable";
+  if (dictionarySearchTruncated) {
+    console.warn("[analytics] Data dictionary search truncated.", {
+      searchedDictionaryEntryCount,
+      dictionarySearchTruncated,
+    });
+  }
+  const favoriteIds =
+    favoritesResult.status === "fulfilled"
+      ? favoritesResult.value
+      : new Set<string>();
+  if (summariesResult.status === "rejected") {
+    warnCatalogReadFailure("Dashboard summary", summariesResult.reason);
+  }
+  if (dictionaryResult.status === "rejected") {
+    warnCatalogReadFailure("Data dictionary", dictionaryResult.reason);
+  }
+  if (favoritesResult.status === "rejected") {
+    warnCatalogReadFailure("Favorite dashboard", favoritesResult.reason);
+  }
   const savedSummaryIds = new Set(
     savedSummaries.map((dashboard) => dashboard.id),
   );
 
   const shortlistedSummaries = shortlistDashboardSummaries(
     args.search,
-    savedSummaries,
+    searchedSummaries,
     args.limit,
     favoriteIds,
   );
   const shortlistedIds = shortlistedSummaries.map((dashboard) => dashboard.id);
+  args.signal?.throwIfAborted();
   const savedDashboardsResult = await loadDashboardCatalogDashboards(
     { email: args.email, orgId: args.orgId },
     shortlistedIds,
-  );
+  ).catch((error: unknown) => {
+    warnCatalogReadFailure("Dashboard detail", error);
+    dashboardSearchStatus = "unavailable";
+    return [];
+  });
+  args.signal?.throwIfAborted();
   const savedDashboards = new Map(
     savedDashboardsResult.map((dashboard) => [dashboard.id, dashboard]),
   );
@@ -707,22 +869,31 @@ export async function searchAnalyticsQueryCatalog(args: {
       }
     });
 
-  return rankAnalyticsQueryCatalog({
-    search: args.search,
-    dashboards: [
-      ...shortlistedSummaries.flatMap((summary) => {
-        const dashboard = savedDashboards.get(summary.id);
-        if (!dashboard) return [];
-        return [
-          savedDashboardInput(
-            { ...summary, favorite: favoriteIds.has(summary.id) },
-            dashboard,
-          ),
-        ];
-      }),
-      ...templateDashboards,
-    ],
-    dictionaryEntries,
-    limit: args.limit,
-  });
+  return {
+    candidates: rankAnalyticsQueryCatalog({
+      search: args.search,
+      dashboards: [
+        ...shortlistedSummaries.flatMap((summary) => {
+          const dashboard = savedDashboards.get(summary.id);
+          if (!dashboard) return [];
+          return [
+            savedDashboardInput(
+              { ...summary, favorite: favoriteIds.has(summary.id) },
+              dashboard,
+            ),
+          ];
+        }),
+        // A truncated window cannot prove a shipped default has no saved version.
+        ...(dashboardSearchTruncated ? [] : templateDashboards),
+      ],
+      dictionaryEntries,
+      limit: args.limit,
+    }),
+    searchedDashboardCount: searchedSummaries.length,
+    dashboardSearchTruncated,
+    dashboardSearchStatus,
+    searchedDictionaryEntryCount,
+    dictionarySearchTruncated,
+    dictionarySearchStatus,
+  };
 }

package/corpus/templates/analytics/server/lib/dashboard-catalog.ts

@@ -931,7 +931,7 @@ export const dashboardCatalogEntries: DashboardCatalogEntry[] = [
     id: "first-party-template-traffic",
     name: "First-party Template Traffic",
     description:
-      "Template signups, activation and onboarding funnel, sharing actions, clicks, demo starts, CLI copies, activity, top URLs, top clips, retention by template, and active users over time.",
+      "Template signups, activation and onboarding step/choice funnels, sharing actions, clicks, demo starts, CLI copies, activity, top URLs, top clips, retention by template, and active users over time.",
     category: "Product",
     defaultDashboardId: "agent-native-templates-first-party",
     dataSources: ["first-party"],
@@ -948,7 +948,7 @@ export const dashboardCatalogEntries: DashboardCatalogEntry[] = [
       "referrals",
       "virality",
     ],
-    panelCount: 42,
+    panelCount: 43,
     version: CATALOG_VERSION,
     recommended: true,
     visibleInCatalog: false,

package/corpus/templates/analytics/server/lib/dashboards-store.ts

@@ -31,6 +31,7 @@ import {
 } from "@agent-native/core/sharing";
 import {
   and,
+  asc,
   desc,
   eq,
   inArray,
@@ -220,18 +221,29 @@ interface AccessCtx {
  */
 async function getScopedLegacySettings(
   ctx: Pick<AccessCtx, "email" | "orgId">,
+  options?: { dashboardKind?: DashboardKind; limit?: number },
 ): Promise<Record<string, Record<string, unknown>>> {
   // User scope first, then org: callers append these to the SQL rows in
   // iteration order and never re-sort, so the order is user-visible. The
   // previous full-table read inherited whatever order the settings table
   // returned, which no query pinned.
   const prefixes: string[] = [];
-  if (ctx.email) prefixes.push(`u:${ctx.email}:`);
-  if (ctx.orgId) prefixes.push(`o:${ctx.orgId}:`);
+  if (options?.dashboardKind === "sql") {
+    if (ctx.email) prefixes.push(`u:${ctx.email}:${SQL_PREFIX}`);
+    if (ctx.orgId) prefixes.push(`o:${ctx.orgId}:${SQL_PREFIX}`);
+  } else {
+    if (ctx.email) prefixes.push(`u:${ctx.email}:`);
+    if (ctx.orgId) prefixes.push(`o:${ctx.orgId}:`);
+  }
   if (prefixes.length === 0) return {};
   const scoped: Record<string, Record<string, unknown>> = {};
   for (const entries of await Promise.all(
-    prefixes.map((prefix) => listSettingsByPrefix(prefix)),
+    prefixes.map((prefix) =>
+      listSettingsByPrefix(
+        prefix,
+        options?.limit === undefined ? undefined : { limit: options.limit },
+      ),
+    ),
   )) {
     for (const { key, value } of entries) scoped[key] = value;
   }
@@ -956,6 +968,7 @@ export async function listDashboardSummaries(
     hidden?: DashboardHiddenFilter;
     includeCatalogMetadata?: boolean;
     legacyScan?: "best-effort" | "strict";
+    limit?: number;
   },
   dbOverride?: any,
 ): Promise<DashboardSummaryRecord[]> {
@@ -963,6 +976,15 @@ export async function listDashboardSummaries(
   const archived = filter?.archived ?? "active";
   const hidden = filter?.hidden ?? "visible";
   const includeCatalogMetadata = filter?.includeCatalogMetadata === true;
+  const summaryLimit = filter?.limit;
+  if (
+    summaryLimit !== undefined &&
+    (!Number.isSafeInteger(summaryLimit) || summaryLimit < 0)
+  ) {
+    throw new RangeError(
+      "Dashboard summary limit must be a non-negative integer.",
+    );
+  }
   const conditions: any[] = [
     accessFilter(schema.dashboards, schema.dashboardShares, {
       userEmail: ctx.email,
@@ -993,7 +1015,7 @@ export async function listDashboardSummaries(
   const demoId = sql<
     string | null
   >`(${schema.dashboards.config}::jsonb -> 'demo' ->> 'id')`;
-  const rows = await db
+  const rowsQuery = db
     .select({
       id: schema.dashboards.id,
       kind: schema.dashboards.kind,
@@ -1016,6 +1038,11 @@ export async function listDashboardSummaries(
     })
     .from(schema.dashboards)
     .where(where);
+  const rows = await (summaryLimit === undefined
+    ? rowsQuery
+    : rowsQuery
+        .orderBy(desc(schema.dashboards.updatedAt), asc(schema.dashboards.id))
+        .limit(summaryLimit));
   const out: DashboardSummaryRecord[] = rows.map((row: any) => {
     const certification = parseDashboardCertification(row.certification);
     const { certification: _rawCertification, ...summaryRow } = row;
@@ -1041,10 +1068,17 @@ export async function listDashboardSummaries(
   });
   const seen = new Set(out.map((row) => row.id));
 
+  if (summaryLimit !== undefined && out.length >= summaryLimit) return out;
   if (archived === "archived" || hidden === "hidden") return out;
   try {
-    const all = await getScopedLegacySettings(ctx);
+    const all = await getScopedLegacySettings(
+      ctx,
+      summaryLimit === undefined
+        ? undefined
+        : { dashboardKind: filter?.kind, limit: summaryLimit },
+    );
     for (const [key, value] of Object.entries(all)) {
+      if (summaryLimit !== undefined && out.length >= summaryLimit) break;
       let id: string | null = null;
       let kind: DashboardKind | null = null;
       let orgId: string | null = null;

package/corpus/templates/analytics/server/lib/data-dictionary-context.ts

@@ -59,6 +59,10 @@ function renderEntry(
 
   const table = compact(entry.table, 240);
   if (table) lines.push(`  - table: ${table}`);
+  const source = compact(entry.source, 240);
+  if (source) lines.push(`  - source: ${source}`);
+  const action = compact(entry.action, 160);
+  if (action) lines.push(`  - query action: ${action}`);
   const columns = compact(entry.columnsUsed, 360);
   if (columns) lines.push(`  - columns: ${columns}`);
   const cuts = compact(entry.cuts, 240);