@ganju/utils 0.0.14 → 0.0.16

package/src/chunking.ts

@@ -1,4 +1,5 @@
 import { constants } from './constants';
+import { overlapTail } from './chunkOverlap';
 import { decodeEntities } from './sanitize';
 
 import type {
@@ -68,7 +69,7 @@ const chunkPieces = (
   const rawChunks = splitRecursive(trimmed, target);
   const pieces: Array<{ raw: string; content: string }> = [];
   for (let i = 0; i < rawChunks.length; i++) {
-    const prev = i > 0 ? rawChunks[i - 1].slice(-overlap) : '';
+    const prev = i > 0 ? overlapTail(rawChunks[i - 1], overlap) : '';
     pieces.push({
       raw: rawChunks[i],
       content: (prev ? prev + '\n' : '') + rawChunks[i]
@@ -93,6 +94,12 @@ interface HeadingEvent {
 const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
 const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
 const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
+// A heading tag still open at the end of a piece, from its opening tag on: the
+// chunker split inside it, and the rest of it opens the next piece.
+const OPEN_HTML_HEADING = /<h([1-6])\b[^>]*>(?![\s\S]*<\/h\1\s*>)[\s\S]*$/i;
+// Longer than any heading worth carrying: an opening tag this far back with no
+// close is broken markup, not a heading that happened to be split.
+const MAX_OPEN_HEADING_CHARS = 1000;
 const MAX_HEADING_CHARS = 120;
 
 const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
@@ -163,16 +170,30 @@ const scanHtmlHeadings = (text: string): HeadingEvent[] =>
 const createHeadingTracker = (syntax: HeadingSyntax) => {
   let stack: Array<{ level: number; text: string }> = [];
   const fence = { inFence: false };
+  // The unfinished HTML heading the previous piece ended in, if any.
+  let carry = '';
   const apply = (event: HeadingEvent) => {
     stack = stack.filter(h => h.level < event.level);
     stack.push({ level: event.level, text: event.text });
   };
 
+  // An HTML heading split between two pieces is read whole from the second,
+  // where it ends, as if that piece opened with it: it does, mid-heading. The
+  // separator the chunker dropped at the split is restored as a space.
+  const scanHtml = (raw: string): HeadingEvent[] => {
+    const text = carry ? `${carry} ${raw}` : raw;
+    const shift = text.length - raw.length;
+    const open = OPEN_HTML_HEADING.exec(text)?.[0] ?? '';
+    carry = open.length <= MAX_OPEN_HEADING_CHARS ? open : '';
+    return scanHtmlHeadings(text).map(event => ({
+      ...event,
+      offset: Math.max(0, event.offset - shift)
+    }));
+  };
+
   return (raw: string): string[] => {
     const events =
-      syntax === 'markdown'
-        ? scanMarkdownHeadings(raw, fence)
-        : scanHtmlHeadings(raw);
+      syntax === 'markdown' ? scanMarkdownHeadings(raw, fence) : scanHtml(raw);
     const [first, ...rest] = events;
     // Before a document's first heading there is no section to continue — only
     // the metadata header, or the markup around an HTML body — so a chunk that

package/src/citations.ts

@@ -0,0 +1,127 @@
+import type { Source } from './sources';
+
+// "p. 125", "pp. 123–126", "page 12", "págs. 4-5", "páginas 3 a 5". An
+// abbreviation counts only with its dot and a number has to stand alone, so
+// "p2p", "mp3" and "pp 2026" are not page references. The lookarounds, not \b,
+// mark the edges: \b sees "á" as a word break.
+const PAGE_REFERENCE =
+  /(?<![\p{L}\p{N}])(?:(?:pp?|p[aá]gs?)\.|pages?|p[aá]ginas?)\s*(\d{1,5})(?![\p{L}\p{N}])(?:(?:\s*[-–—]\s*|\s+(?:to|a)\s+)(\d{1,5})(?![\p{L}\p{N}]))?/giu;
+const MAX_PAGE_RANGE = 50;
+
+// Lowercased words only, so "ETag header — HTTP | MDN" in an answer matches the
+// title "ETag header - HTTP | MDN", and "mml-book.pdf" matches "mml-book".
+const normalize = (text: string): string =>
+  ` ${text
+    .toLowerCase()
+    .normalize('NFKD')
+    .replace(/[̀-ͯ]/g, '')
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()} `;
+
+// The ways an answer refers to a source: its whole title, the title without a
+// parenthetical or a site suffix ("Northwind Handbook (draft)", "ETag header
+// - HTTP | MDN"), its file name with and without the extension, and its URL.
+const namesFor = (source: Source): string[] => {
+  const names = [source.title, source.title.replace(/\s*\([^)]*\)\s*/g, ' ')];
+  const lead = source.title.split(/\s+[|\-–—:]\s+/)[0];
+  // A one-word lead ("Introduction", "Pricing") names half the corpus.
+  if (lead.trim().split(/\s+/).length >= 2) names.push(lead);
+  if (source.fileName) {
+    names.push(source.fileName, source.fileName.replace(/\.[a-z0-9]+$/i, ''));
+  }
+  return names.map(normalize).filter(name => name.trim().length >= 4);
+};
+
+// The characters that can carry a URL on past the point we matched. Brackets,
+// quotes and whitespace end it — that is how a URL sits inside Markdown links,
+// parentheses and prose.
+const URL_CONTINUATION = /^[^\s<>()[\]{}"'`|]*/;
+// Punctuation that closes a sentence around a URL rather than belonging to it.
+const TRAILING_PUNCTUATION = /[.,;:!?*_]+$/;
+
+/**
+ * Whether the answer cites this exact URL, not merely one that starts with it:
+ * on a crawled docs site the index page is a prefix of every page beneath it,
+ * so a plain substring test kept "/docs" and the site root in the footer of an
+ * answer that cited "/docs/api". A trailing slash and a #fragment still count
+ * as the same page.
+ */
+export const citesUrl = (answer: string, url: string): boolean => {
+  const target = url.replace(/\/+$/, '');
+  if (!target) return false;
+  for (
+    let at = answer.indexOf(target);
+    at !== -1;
+    at = answer.indexOf(target, at + 1)
+  ) {
+    const rest = answer
+      .slice(at + target.length)
+      .match(URL_CONTINUATION)![0]
+      .replace(TRAILING_PUNCTUATION, '');
+    if (/^\/*(#.*)?$/.test(rest)) return true;
+  }
+  return false;
+};
+
+/** Every page number the answer cites, ranges expanded. */
+export const citedPages = (answer: string): Set<number> => {
+  const pages = new Set<number>();
+  for (const match of answer.matchAll(PAGE_REFERENCE)) {
+    const from = Number(match[1]);
+    const to = match[2] ? Number(match[2]) : from;
+    if (to < from || to - from > MAX_PAGE_RANGE) {
+      pages.add(from);
+      continue;
+    }
+    for (let page = from; page <= to; page++) pages.add(page);
+  }
+  return pages;
+};
+
+/**
+ * Narrow a turn's sources to the ones its answer actually cites.
+ *
+ * Every search hit used to become a footer line, so an answer drawn from one
+ * handbook section was followed by eight unrelated pages the search had also
+ * returned. A search hit is kept when the answer names its resource — by title,
+ * file name or URL — and, for a paged file, only the pages the answer cites
+ * (or the best-ranked one, when it cites none). Resources the model read whole
+ * are always kept: reading one is itself the citation.
+ *
+ * An answer that names nothing gets no search sources at all. The model is told
+ * to cite what it uses, so a reply that names nothing is a greeting or "I
+ * couldn't find that" — and a footer under it would claim sources it never
+ * used, just because search runs on every message.
+ */
+export const selectCitedSources = (
+  sources: Source[],
+  answerText: string
+): Source[] => {
+  const searchEntries = sources.filter(s => s.score !== undefined);
+  const readEntries = sources.filter(s => s.score === undefined);
+  if (searchEntries.length === 0) return sources;
+
+  const answer = normalize(answerText);
+  const pages = citedPages(answerText);
+  const citedResources = new Set(
+    searchEntries
+      .filter(
+        s =>
+          citesUrl(answerText, s.uri) ||
+          namesFor(s).some(name => answer.includes(name))
+      )
+      .map(s => s.resourceId)
+  );
+
+  const kept: Source[] = [];
+  for (const resourceId of citedResources) {
+    const entries = searchEntries.filter(s => s.resourceId === resourceId);
+    const onCitedPage = entries.filter(
+      s => s.pageNumber !== undefined && pages.has(s.pageNumber)
+    );
+    kept.push(...(onCitedPage.length > 0 ? onCitedPage : entries.slice(0, 1)));
+  }
+
+  const cited = searchEntries.filter(s => kept.includes(s));
+  return [...cited, ...readEntries];
+};

package/src/constants.ts

@@ -144,6 +144,11 @@ const OAUTH_TOKEN_URLS: Record<string, string> = {
 // MCP-proxy OAuth, in both apps/api and apps/mcp).
 const CREDENTIAL_REFRESH_BUFFER_MS = 60 * 1000;
 
+// A refresh the token endpoint rejects (4xx) without a reauth code still flags
+// the credential once its access token has been expired this long, so the
+// owner sees it as expired instead of every call failing behind "Connected".
+const CREDENTIAL_REFRESH_GIVE_UP_MS = 60 * 60 * 1000;
+
 const GOOGLE_CALENDAR_API_BASE = 'https://www.googleapis.com/calendar/v3';
 
 const GOOGLE_DRIVE_API_BASE = 'https://www.googleapis.com/drive/v3';
@@ -228,7 +233,18 @@ const ONE_DRIVE_TAB_LABELS: ReadonlyArray<{
   { value: ONE_DRIVE_TAB_DRIVES, label: ONE_DRIVE_TAB_LABEL_DRIVES }
 ];
 
-const REAUTH_ERROR_CODES = ['invalid_grant', 'invalid_token'];
+// Token-endpoint error codes that mean the stored refresh token is dead and
+// only a reconnect brings it back. `unauthorized_client` is Google's answer
+// when a token no longer belongs to the app it is refreshed with.
+const REAUTH_ERROR_CODES = [
+  'invalid_grant',
+  'invalid_token',
+  'unauthorized_client'
+];
+
+// Token-endpoint error codes that are about our own client id / secret, not
+// the user's token: a reconnect can't fix them, so they never flag one.
+const OAUTH_CLIENT_ERROR_CODES = ['invalid_client'];
 
 const RESOURCE_TYPE_TOOLTIP_MIN_WIDTH = 300;
 
@@ -324,6 +340,9 @@ const CRAWL_MAX_PAGES_LIMIT = 1000;
 const CRAWL_DEFAULT_MAX_DEPTH = 3;
 const CRAWL_MAX_DEPTH_LIMIT = 10;
 const CRAWL_PAGE_FETCH_TIMEOUT_MS = 30000;
+// Redirects one crawl fetch may follow. Each hop is screened on its own, so a
+// public URL can't bounce the crawler onto a private address.
+const CRAWL_MAX_REDIRECTS = 5;
 const CRAWL_USER_AGENT =
   'Mozilla/5.0 (compatible; GanjuCrawler/1.0; +https://ganju.ai)';
 const CRAWL_PAGE_QUEUE_BATCH_SIZE = 100;
@@ -597,6 +616,11 @@ const RATE_LIMIT_BACKOFF_SECONDS = 60;
 // / Container reset) so the retried job lands after the object has recovered.
 const TRANSIENT_BACKOFF_SECONDS = 5;
 
+// Retries every resource queue consumer is configured with (`max_retries` in
+// the API worker's wrangler config). A job's last delivery is attempt
+// QUEUE_MAX_RETRIES + 1; only then is its work marked failed.
+const QUEUE_MAX_RETRIES = 3;
+
 const CHANNEL_PLATFORM_TELEGRAM = 'telegram' as 'telegram';
 const CHANNEL_PLATFORM_SLACK = 'slack' as 'slack';
 const CHANNEL_PLATFORM_WHATSAPP = 'whatsapp' as 'whatsapp';
@@ -808,15 +832,39 @@ const TOOL_CONFIRMATION_ASK_INSTRUCTION =
   'confirm, and must not say or suggest that they were done, nor describe ' +
   'or make up any result from them.';
 // For MCP clients, whose chat the server can't see: a sensitive call without a
-// valid confirmation isn't run, and the model is told to ask the user and call
-// again with a token that binds the second call to these exact arguments.
-// How long that token stays good, and the argument that carries it.
+// valid confirmation isn't run. Ordinarily the model is told to ask the user
+// and call again with a token that binds the second call to these exact
+// arguments; once the session has read outside content, the user approves it
+// on a dashboard page instead, and the second call carries that approval's
+// id. How long either stays good, and the argument that carries it.
 const MCP_TOOL_CONFIRMATION_TTL_MS = 10 * 60 * 1000;
 const MCP_TOOL_CONFIRMATION_ARG = 'confirmation';
 const MCP_TOOL_CONFIRMATION_ARG_DESCRIPTION =
   "Leave out on the first call. This action needs the user's yes: the " +
-  'first call returns a token; after the user says yes, call again with the ' +
-  'same arguments and that token here.';
+  'first call returns a code and says how to get that yes; once the user ' +
+  'has given it, call again with the same arguments and that code here.';
+// The same argument where only some calls ask: those made after the session
+// read outside content.
+const MCP_TOOL_CONFIRMATION_ARG_DESCRIPTION_CONDITIONAL =
+  "Leave out on the first call. If this action needs the user's yes, the " +
+  'first call returns a code instead of running and says how to get that ' +
+  'yes; once the user has given it, call again with the same arguments and ' +
+  'that code here.';
+// The dashboard page where a signed-in user approves or denies one such call,
+// as `${TOOL_APPROVAL_PATH}/<approval id>`, and the two answers it sends.
+const TOOL_APPROVAL_PATH = '/tool-approval';
+const TOOL_APPROVAL_DECISION_APPROVE = 'approve';
+const TOOL_APPROVAL_DECISION_DENY = 'deny';
+const TOOL_APPROVAL_DECISIONS = [
+  TOOL_APPROVAL_DECISION_APPROVE,
+  TOOL_APPROVAL_DECISION_DENY
+] as const;
+// Where an approval stands, as the page shows it.
+const TOOL_APPROVAL_STATUS_PENDING = 'pending';
+const TOOL_APPROVAL_STATUS_APPROVED = 'approved';
+const TOOL_APPROVAL_STATUS_DENIED = 'denied';
+const TOOL_APPROVAL_STATUS_USED = 'used';
+const TOOL_APPROVAL_STATUS_EXPIRED = 'expired';
 // The question when the model wrote none, above the runner's own summary.
 const TOOL_CONFIRMATION_FALLBACK_QUESTION =
   'Before I do this, please confirm. Reply yes to go ahead, or no to cancel.';
@@ -841,13 +889,15 @@ const SLACK_MAX_SOURCE_BUTTONS = 10;
 
 // The Slack app configuration Ganju needs, surfaced in the UI when connecting a
 // Slack channel. Required scopes back the runner's calls (mentions, DMs,
-// posting, file upload); recommended scopes only enrich names/titles and
-// degrade gracefully; bot events are what the webhook subscribes to.
+// posting, file upload, downloading a voice clip to transcribe it);
+// recommended scopes only enrich names/titles and degrade gracefully; bot
+// events are what the webhook subscribes to.
 const SLACK_REQUIRED_SCOPES = [
   'app_mentions:read',
   'im:history',
   'chat:write',
-  'files:write'
+  'files:write',
+  'files:read'
 ];
 const SLACK_RECOMMENDED_SCOPES = ['users:read', 'channels:read', 'groups:read'];
 const SLACK_BOT_EVENTS = ['app_mention', 'message.im'];
@@ -958,12 +1008,29 @@ const RESOURCE_SEARCH_RRF_K = 60;
 // keeps its words.
 const RESOURCE_SEARCH_COMMON_TERM_DIVISOR = 10;
 const RESOURCE_SEARCH_COMMON_TERM_MIN = 5;
-// Second stage, on wherever the Worker has an AI binding: the top fused
-// candidates are re-scored by a cross-encoder, which reads query and chunk
-// together instead of comparing two independently computed vectors. One model
-// call per search; a failure falls back to the fused order.
+// Second stage: the top fused candidates re-scored by a cross-encoder, which
+// reads query and chunk together instead of comparing two independently
+// computed vectors. One model call per search; a failure falls back to the
+// fused order. Off by default: on the golden sets (Spanish and English) the
+// fused order matched or beat it on every set — 10 cases better, 5 worse — and
+// it added 0.5–0.9 s a search plus a ~6 s cold start. The search eval can
+// still turn it on per request to re-measure.
+const RESOURCE_SEARCH_RERANK_ENABLED = false;
 const RESOURCE_SEARCH_RERANK_CANDIDATES = 30;
 const RESOURCE_SEARCH_RERANK_MODEL = '@cf/baai/bge-reranker-base';
+// An identifier in the query (a code mixing letters and digits, a snake_case
+// name, or a number, dotted or hyphenated name typed on its own) found in at
+// most this many documents is what the user is after: its chunks go first,
+// whatever the reranker thinks. A cross-encoder reads "86400" or "PGRST116" as noise and
+// would bury the one chunk that contains it under pages that merely look
+// related. Past EXACT_MAX_CHUNKS matching chunks the identifier is common
+// enough that the usual ranking is left to decide, and the scan stops there.
+const RESOURCE_SEARCH_EXACT_MAX_RESOURCES = 5;
+const RESOURCE_SEARCH_EXACT_MAX_CHUNKS = 30;
+// An identifier the index never saw is looked for as plain text, which reads
+// every chunk of the artifact when nothing matches. Past this many chunks that
+// costs more than the rare hit is worth, so a large artifact skips it.
+const RESOURCE_SEARCH_SUBSTRING_MAX_CHUNKS = 20000;
 
 const RESOURCE_HANDLER_SLEEP_AFTER = '10m';
 
@@ -983,6 +1050,12 @@ const CHUNK_SEPARATORS: Array<{ split: string | RegExp; join: string }> = [
 
 const DEFAULT_MAX_TOKENS = 4096;
 
+// One page of `list-resources`. A knowledge base with a crawled site runs to
+// thousands of resources; listed whole, that is one result of several hundred
+// KB in the model's context.
+const LIST_RESOURCES_DEFAULT_LIMIT = 100;
+const LIST_RESOURCES_MAX_LIMIT = 500;
+
 const RESOURCE_TOOL_KEY_LIST_RESOURCES = 'list-resources';
 const RESOURCE_TOOL_KEY_SEARCH_RESOURCES = 'search-resources';
 const RESOURCE_TOOL_KEY_READ_RESOURCE = 'read-resource';
@@ -1514,7 +1587,9 @@ const RESERVED_TOOL_NAMES = [
   TOOL_DEFINITION_KEY_HTTP_ENDPOINT,
   TOOL_DEFINITION_KEY_MCP_PROXY,
   TOOL_DEFINITION_KEY_CUSTOM_CODE,
-  CUSTOM_CODE_HEALTH_TOOL
+  CUSTOM_CODE_HEALTH_TOOL,
+  // The channel runner's own memory tool (PARTICIPANT_MEMORY_TOOL_NAME).
+  'remember-customer'
 ];
 
 // Shared by both write paths so there is one string to translate. It has to
@@ -1779,8 +1854,16 @@ const MCP_AUTH_KINDS = [MCP_AUTH_KIND_JWT, MCP_AUTH_KIND_INTERNAL];
 const MCP_SESSION_HEADER = 'mcp-session-id';
 
 const MCP_INTERNAL_HEADER = 'x-ganju-internal-secret';
+// Sent with the internal secret by the search eval to measure the reranker:
+// `on` or `off` overrides RESOURCE_SEARCH_RERANK_ENABLED for that request.
+// Ignored on any other request.
+const MCP_EVAL_RERANK_HEADER = 'x-ganju-eval-rerank';
 const MCP_CHANNEL_ID_HEADER = 'x-ganju-channel-id';
 const MCP_CHANNEL_PLATFORM_HEADER = 'x-ganju-channel-platform';
+// The conversation a channel turn belongs to. The MCP worker files that turn's
+// calls under one session per conversation, so a call can be traced back to
+// the chat it ran in.
+const MCP_CHANNEL_CONVERSATION_HEADER = 'x-ganju-channel-conversation-id';
 const MCP_CHANNEL_CLIENT_USER_AGENT = 'ganju-channel/0.0.1';
 const JWKS_KV_KEY = 'jwks:v1';
 const JWKS_TTL_SECONDS = 600;
@@ -1882,6 +1965,23 @@ const ACCESS_TOKEN_SCOPE_MESSAGE =
 const CUSTOM_CODE_LOGS_DEFAULT_LIMIT = 20;
 const CUSTOM_CODE_LOGS_MAX_LIMIT = 100;
 
+// Tool observability on the project home. The windows match the activity
+// chart's, and none reaches past `RETENTION_DAYS.mcpRequest`.
+const OBSERVABILITY_RANGE_DAYS = [7, 30, 90];
+// An enabled tool with no call in this many days is offered for disabling.
+const OBSERVABILITY_UNUSED_DAYS = 30;
+// Two calls in one session this close together are one attempt followed by
+// the next: a retry when it's the same tool, a switch when it's another tool
+// after an error.
+const OBSERVABILITY_FOLLOW_UP_SECONDS = 120;
+// How a call rejected by its tool's input schema reads in `error_message`.
+const OBSERVABILITY_SCHEMA_REJECTION_PREFIX = 'Input validation error';
+const OBSERVABILITY_TOP_ERRORS = 8;
+const OBSERVABILITY_CALLS_DEFAULT_LIMIT = 25;
+const OBSERVABILITY_CALLS_MAX_LIMIT = 100;
+// A session timeline is read whole up to this many calls, newest kept.
+const OBSERVABILITY_SESSION_MAX_CALLS = 500;
+
 const BOT_GRANT_TYPE = 'urn:ganju:bot-on-behalf-of';
 const EXTERNAL_LINK_VERIFICATION_PREFIX = 'external_link:';
 const EXTERNAL_LINK_TTL_SECONDS = 600;
@@ -2051,6 +2151,40 @@ const PRICING_TOOL_CALL_PER_M_USD = 5;
 const PRICING_TOOL_CALL_HARD_CAP = 20_000_000;
 const PRICING_CUSTOM_DOMAIN_USD = 15;
 
+// Project data (one SQLite Durable Object per artifact). Cloudflare caps a
+// database at 10 GB; we stop writes that add data at 9 GB so there is room to
+// delete, archive and export.
+const DATA_MAX_DATABASE_BYTES = 9 * GB;
+const DATA_MAX_TABLES_PER_DATABASE = 50;
+const DATA_TRIGGER_RUN_HARD_CAP = 5_000_000;
+// A deleted database can be restored for this long before its storage is
+// freed.
+const DATA_DELETE_GRACE_DAYS = 7;
+const ARTIFACT_DATABASE_STATUS_CREATING = 'CREATING' as 'CREATING';
+const ARTIFACT_DATABASE_STATUS_READY = 'READY' as 'READY';
+const ARTIFACT_DATABASE_STATUS_FAILED = 'FAILED' as 'FAILED';
+const ARTIFACT_DATABASE_STATUS_DELETING = 'DELETING' as 'DELETING';
+// The hourly purge has claimed the row and is emptying the object; a restore
+// can no longer take it back.
+const ARTIFACT_DATABASE_STATUS_PURGING = 'PURGING' as 'PURGING';
+const ARTIFACT_DATABASE_STATUS_DELETED = 'DELETED' as 'DELETED';
+const ARTIFACT_DATABASE_STATUSES = [
+  ARTIFACT_DATABASE_STATUS_CREATING,
+  ARTIFACT_DATABASE_STATUS_READY,
+  ARTIFACT_DATABASE_STATUS_FAILED,
+  ARTIFACT_DATABASE_STATUS_DELETING,
+  ARTIFACT_DATABASE_STATUS_PURGING,
+  ARTIFACT_DATABASE_STATUS_DELETED
+];
+// A purge that claimed a row and never finished (the Worker died) is picked
+// up again after this long.
+const DATA_PURGE_STALE_MS = 60 * 60 * 1000;
+// Rows purged per hourly run: each is a call to its database object.
+const DATA_PURGE_BATCH = 20;
+// last_read_at / last_write_at move at most this often, so they aren't a
+// Postgres write per query.
+const DATA_ACTIVITY_THROTTLE_MS = 60 * 60 * 1000;
+
 interface PlanLimits {
   // null on any field means "no hard limit" for that plan.
   maxProjects: number | null;
@@ -2116,6 +2250,16 @@ interface PlanLimits {
   autoSyncIntervals: ReadonlyArray<
     typeof RESOURCE_SYNC_INTERVAL_DAILY | typeof RESOURCE_SYNC_INTERVAL_WEEKLY
   >;
+  // Whether a project may create a database (tables and rows). Paid-only: the
+  // rows live in a Durable Object we pay storage and rows for. An org that
+  // downgrades keeps its databases, read-only.
+  canUseData: boolean;
+  // Writes that add data stop here, under Cloudflare's 10 GB per database, so
+  // the owner meets a readable "database full" rather than SQLITE_FULL.
+  maxDataBytes: number | null;
+  maxDataTablesPerDatabase: number | null;
+  // Abuse backstop on data trigger runs a month. `null` = no backstop.
+  triggerRunHardCap: number | null;
 }
 
 const PLAN_LIMITS: Record<
@@ -2156,7 +2300,11 @@ const PLAN_LIMITS: Record<
     toolCallHardCap: 10_000,
     includedMessages: 100,
     includedEmbeddedBytes: 5 * MB,
-    autoSyncIntervals: []
+    autoSyncIntervals: [],
+    canUseData: false,
+    maxDataBytes: null,
+    maxDataTablesPerDatabase: null,
+    triggerRunHardCap: 0
   },
   PRO: {
     maxProjects: null,
@@ -2183,7 +2331,11 @@ const PLAN_LIMITS: Record<
     autoSyncIntervals: [
       RESOURCE_SYNC_INTERVAL_DAILY,
       RESOURCE_SYNC_INTERVAL_WEEKLY
-    ]
+    ],
+    canUseData: true,
+    maxDataBytes: DATA_MAX_DATABASE_BYTES,
+    maxDataTablesPerDatabase: DATA_MAX_TABLES_PER_DATABASE,
+    triggerRunHardCap: DATA_TRIGGER_RUN_HARD_CAP
   },
   ENTERPRISE: {
     maxProjects: null,
@@ -2210,7 +2362,11 @@ const PLAN_LIMITS: Record<
     autoSyncIntervals: [
       RESOURCE_SYNC_INTERVAL_DAILY,
       RESOURCE_SYNC_INTERVAL_WEEKLY
-    ]
+    ],
+    canUseData: true,
+    maxDataBytes: DATA_MAX_DATABASE_BYTES,
+    maxDataTablesPerDatabase: DATA_MAX_TABLES_PER_DATABASE,
+    triggerRunHardCap: null
   }
 };
 
@@ -2230,6 +2386,7 @@ const PLAN_FEATURE_EMBEDDED_STORAGE = 'embeddedStorage' as 'embeddedStorage';
 const PLAN_FEATURE_MESSAGE = 'message' as 'message';
 const PLAN_FEATURE_TOOL_CALL = 'toolCall' as 'toolCall';
 const PLAN_FEATURE_RESOURCE_SYNC = 'resourceSync' as 'resourceSync';
+const PLAN_FEATURE_DATA = 'data' as 'data';
 
 // Stable code returned on a quota block so clients can branch on it (402).
 const PLAN_LIMIT_ERROR_CODE = 'PLAN_LIMIT_EXCEEDED';
@@ -2399,6 +2556,20 @@ export const constants = {
   PRICING_TOOL_CALL_PER_M_USD,
   PRICING_TOOL_CALL_HARD_CAP,
   PRICING_CUSTOM_DOMAIN_USD,
+  DATA_MAX_DATABASE_BYTES,
+  DATA_MAX_TABLES_PER_DATABASE,
+  DATA_TRIGGER_RUN_HARD_CAP,
+  DATA_DELETE_GRACE_DAYS,
+  DATA_PURGE_BATCH,
+  DATA_ACTIVITY_THROTTLE_MS,
+  ARTIFACT_DATABASE_STATUS_CREATING,
+  ARTIFACT_DATABASE_STATUS_READY,
+  ARTIFACT_DATABASE_STATUS_FAILED,
+  ARTIFACT_DATABASE_STATUS_DELETING,
+  ARTIFACT_DATABASE_STATUS_PURGING,
+  ARTIFACT_DATABASE_STATUS_DELETED,
+  DATA_PURGE_STALE_MS,
+  ARTIFACT_DATABASE_STATUSES,
   PLAN_FEATURE_ORGANIZATION,
   PLAN_FEATURE_PROJECT,
   PLAN_FEATURE_TOOL,
@@ -2413,6 +2584,7 @@ export const constants = {
   PLAN_FEATURE_MESSAGE,
   PLAN_FEATURE_TOOL_CALL,
   PLAN_FEATURE_RESOURCE_SYNC,
+  PLAN_FEATURE_DATA,
   PLAN_LIMIT_ERROR_CODE,
   BILLING_METER_MESSAGES,
   BILLING_METER_SHARED_MESSAGES,
@@ -2501,6 +2673,7 @@ export const constants = {
   SLACK_OAUTH_TOKEN_URL,
   OAUTH_TOKEN_URLS,
   CREDENTIAL_REFRESH_BUFFER_MS,
+  CREDENTIAL_REFRESH_GIVE_UP_MS,
   GOOGLE_CALENDAR_API_BASE,
   GOOGLE_DRIVE_API_BASE,
   GOOGLE_DRIVE_DEFAULT_PAGE_SIZE,
@@ -2536,6 +2709,7 @@ export const constants = {
   ONE_DRIVE_TAB_LABEL_DRIVES,
   ONE_DRIVE_TAB_LABELS,
   REAUTH_ERROR_CODES,
+  OAUTH_CLIENT_ERROR_CODES,
   RESOURCE_TYPE_TOOLTIP_MIN_WIDTH,
   SEARCH_DEBOUNCE_MS,
   GOOGLE_DRIVE_EXPORT_MIME_TYPES,
@@ -2624,6 +2798,7 @@ export const constants = {
   CRAWL_DEFAULT_MAX_DEPTH,
   CRAWL_MAX_DEPTH_LIMIT,
   CRAWL_PAGE_FETCH_TIMEOUT_MS,
+  CRAWL_MAX_REDIRECTS,
   CRAWL_PAGE_QUEUE_BATCH_SIZE,
   CRAWL_USER_AGENT,
   RESOURCE_ICON_THEME_DARK,
@@ -2641,6 +2816,7 @@ export const constants = {
   TELEGRAM_MAX_FILE_BYTES,
   RATE_LIMIT_BACKOFF_SECONDS,
   TRANSIENT_BACKOFF_SECONDS,
+  QUEUE_MAX_RETRIES,
   ENCODINGS,
   ENCODING_UTF8,
   CHANNEL_PLATFORM_TELEGRAM,
@@ -2682,6 +2858,16 @@ export const constants = {
   MCP_TOOL_CONFIRMATION_TTL_MS,
   MCP_TOOL_CONFIRMATION_ARG,
   MCP_TOOL_CONFIRMATION_ARG_DESCRIPTION,
+  MCP_TOOL_CONFIRMATION_ARG_DESCRIPTION_CONDITIONAL,
+  TOOL_APPROVAL_PATH,
+  TOOL_APPROVAL_DECISION_APPROVE,
+  TOOL_APPROVAL_DECISION_DENY,
+  TOOL_APPROVAL_DECISIONS,
+  TOOL_APPROVAL_STATUS_PENDING,
+  TOOL_APPROVAL_STATUS_APPROVED,
+  TOOL_APPROVAL_STATUS_DENIED,
+  TOOL_APPROVAL_STATUS_USED,
+  TOOL_APPROVAL_STATUS_EXPIRED,
   CHANNEL_DEBOUNCE_DEFAULT_MS,
   CHANNEL_DEBOUNCE_MIN_MS,
   CHANNEL_DEBOUNCE_MAX_MS,
@@ -2752,8 +2938,12 @@ export const constants = {
   RESOURCE_SEARCH_RRF_K,
   RESOURCE_SEARCH_COMMON_TERM_DIVISOR,
   RESOURCE_SEARCH_COMMON_TERM_MIN,
+  RESOURCE_SEARCH_RERANK_ENABLED,
   RESOURCE_SEARCH_RERANK_CANDIDATES,
   RESOURCE_SEARCH_RERANK_MODEL,
+  RESOURCE_SEARCH_EXACT_MAX_RESOURCES,
+  RESOURCE_SEARCH_EXACT_MAX_CHUNKS,
+  RESOURCE_SEARCH_SUBSTRING_MAX_CHUNKS,
   RESOURCE_HANDLER_SLEEP_AFTER,
   DOCS_URL,
   BASE64_DATA_URI_RE,
@@ -2761,6 +2951,8 @@ export const constants = {
   CHUNK_SEPARATORS,
   DEFAULT_MAX_TOKENS,
   RESOURCE_TOOL_KEY_LIST_RESOURCES,
+  LIST_RESOURCES_DEFAULT_LIMIT,
+  LIST_RESOURCES_MAX_LIMIT,
   RESOURCE_TOOL_KEY_SEARCH_RESOURCES,
   RESOURCE_TOOL_KEY_READ_RESOURCE,
   RESOURCE_TOOL_KEY_SEND_RESOURCE,
@@ -2932,8 +3124,10 @@ export const constants = {
   CUSTOM_CODE_RESOURCE_FILE_TOO_LARGE_MESSAGE,
   RESERVED_SLUGS,
   MCP_INTERNAL_HEADER,
+  MCP_EVAL_RERANK_HEADER,
   MCP_CHANNEL_ID_HEADER,
   MCP_CHANNEL_PLATFORM_HEADER,
+  MCP_CHANNEL_CONVERSATION_HEADER,
   MCP_CHANNEL_CLIENT_USER_AGENT,
   MCP_SESSION_HEADER,
   MCP_REQUEST_METHOD_INITIALIZE,
@@ -2971,6 +3165,14 @@ export const constants = {
   ACCESS_TOKEN_SCOPE_MESSAGE,
   CUSTOM_CODE_LOGS_DEFAULT_LIMIT,
   CUSTOM_CODE_LOGS_MAX_LIMIT,
+  OBSERVABILITY_RANGE_DAYS,
+  OBSERVABILITY_UNUSED_DAYS,
+  OBSERVABILITY_FOLLOW_UP_SECONDS,
+  OBSERVABILITY_SCHEMA_REJECTION_PREFIX,
+  OBSERVABILITY_TOP_ERRORS,
+  OBSERVABILITY_CALLS_DEFAULT_LIMIT,
+  OBSERVABILITY_CALLS_MAX_LIMIT,
+  OBSERVABILITY_SESSION_MAX_CALLS,
   BOT_GRANT_TYPE,
   EXTERNAL_LINK_VERIFICATION_PREFIX,
   EXTERNAL_LINK_TTL_SECONDS,