@ganju/utils 0.0.14 → 0.0.16

package/src/jsonSchemaToZodShape.ts

@@ -1,7 +1,7 @@
 import * as z from 'zod';
 
 export type JsonSchemaProperty = {
-  type: 'string' | 'number' | 'boolean' | 'object' | 'array';
+  type: 'string' | 'number' | 'integer' | 'boolean' | 'object' | 'array';
   description?: string;
   minimum?: number;
   maximum?: number;
@@ -10,6 +10,10 @@ export type JsonSchemaProperty = {
   pattern?: string;
   enum?: string[];
   items?: JsonSchemaProperty;
+  minItems?: number;
+  maxItems?: number;
+  properties?: Record<string, JsonSchemaProperty>;
+  required?: string[];
 };
 
 export type JsonSchema = {
@@ -35,8 +39,9 @@ const propertyToZod = (prop: JsonSchemaProperty): z.ZodTypeAny => {
       }
       break;
     }
-    case 'number': {
-      let num = z.number();
+    case 'number':
+    case 'integer': {
+      let num = prop.type === 'integer' ? z.number().int() : z.number();
       if (prop.minimum !== undefined) num = num.min(prop.minimum);
       if (prop.maximum !== undefined) num = num.max(prop.maximum);
       field = num;
@@ -45,11 +50,21 @@ const propertyToZod = (prop: JsonSchemaProperty): z.ZodTypeAny => {
     case 'boolean':
       field = z.boolean();
       break;
-    case 'array':
-      field = z.array(prop.items ? propertyToZod(prop.items) : z.any());
+    case 'array': {
+      let arr = z.array(prop.items ? propertyToZod(prop.items) : z.any());
+      if (prop.minItems !== undefined) arr = arr.min(prop.minItems);
+      if (prop.maxItems !== undefined) arr = arr.max(prop.maxItems);
+      field = arr;
       break;
+    }
     case 'object':
-      field = z.record(z.string(), z.any());
+      // With declared properties, keep them: this shape is also what
+      // `tools/list` advertises, and a bare record there tells the model an
+      // item is any object at all, so it invents the field names. Loose,
+      // because a schema without `additionalProperties: false` allows extras.
+      field = prop.properties
+        ? z.looseObject(shapeOf(prop.properties, prop.required))
+        : z.record(z.string(), z.any());
       break;
     default:
       field = z.any();
@@ -62,12 +77,10 @@ const propertyToZod = (prop: JsonSchemaProperty): z.ZodTypeAny => {
   return field;
 };
 
-export const jsonSchemaToZodShape = (
-  schema: JsonSchema
-): Record<string, z.ZodTypeAny> => {
-  const properties = schema.properties ?? {};
-  const required = schema.required ?? [];
-
+function shapeOf(
+  properties: Record<string, JsonSchemaProperty> = {},
+  required: string[] = []
+): Record<string, z.ZodTypeAny> {
   const shape: Record<string, z.ZodTypeAny> = {};
 
   for (const [key, prop] of Object.entries(properties)) {
@@ -76,7 +89,12 @@ export const jsonSchemaToZodShape = (
   }
 
   return shape;
-};
+}
+
+export const jsonSchemaToZodShape = (
+  schema: JsonSchema
+): Record<string, z.ZodTypeAny> =>
+  shapeOf(schema.properties, schema.required);
 
 export interface SchemaViolation {
   // Dotted path to the offending field, empty for the value as a whole.

package/src/oauth.ts

@@ -24,6 +24,21 @@ export class OAuthReauthRequiredError extends Error {
   }
 }
 
+// The token endpoint answered with an error that isn't a reauth code. `status`
+// tells a rejection (4xx) from an outage (5xx); a network failure never gets
+// here, it throws fetch's own error.
+export class OAuthRefreshError extends Error {
+  readonly status: number;
+  readonly code: string | undefined;
+
+  constructor(status: number, code: string | undefined, body: string) {
+    super(`oauth refresh failed (${status}): ${body}`);
+    this.name = 'OAuthRefreshError';
+    this.status = status;
+    this.code = code;
+  }
+}
+
 const parseErrorCode = (body: string): string | undefined => {
   try {
     return (JSON.parse(body) as { error?: string })?.error;
@@ -52,7 +67,7 @@ export const refreshOAuthToken = async (
     if (code && constants.REAUTH_ERROR_CODES.includes(code)) {
       throw new OAuthReauthRequiredError(code);
     }
-    throw new Error(`oauth refresh failed (${response.status}): ${body}`);
+    throw new OAuthRefreshError(response.status, code, body);
   }
 
   const tokens = (await response.json()) as {
@@ -95,6 +110,32 @@ export const clearReauthMetadata = (
   return Object.keys(next).length > 0 ? next : null;
 };
 
+/**
+ * Why a failed refresh should flag the credential for a reconnect, or null
+ * when it shouldn't. A reauth code flags it at once. Any other rejection
+ * (4xx) flags it only once the access token has been expired for
+ * CREDENTIAL_REFRESH_GIVE_UP_MS, so a one-off doesn't. Never flagged: an
+ * outage (5xx), a network error, rate limiting (429), and errors about our
+ * own client id / secret, which no reconnect can fix.
+ */
+export const refreshFailureReauthReason = (
+  err: unknown,
+  expiresAt: Date | null,
+  now = Date.now()
+): string | null => {
+  if (err instanceof OAuthReauthRequiredError) return err.code;
+  if (!(err instanceof OAuthRefreshError)) return null;
+  if (err.status < 400 || err.status >= 500 || err.status === 429) return null;
+  if (err.code && constants.OAUTH_CLIENT_ERROR_CODES.includes(err.code)) {
+    return null;
+  }
+  if (!expiresAt) return null;
+  if (now - expiresAt.getTime() < constants.CREDENTIAL_REFRESH_GIVE_UP_MS) {
+    return null;
+  }
+  return err.code ? `refresh_rejected:${err.code}` : 'refresh_rejected';
+};
+
 export const isCredentialNeedingReauth = (metadata: unknown): boolean => {
   if (!metadata || typeof metadata !== 'object') return false;
   return (metadata as Record<string, unknown>).needsReauth === true;

package/src/pagination.ts

@@ -0,0 +1,193 @@
+// Large tool results, in one shape: a list a tool owns is paged by a cursor the
+// model passes back, and text that has to be cut says so, how much was left
+// out, and where to continue when the result itself carried a cursor.
+//
+// Imports nothing, so node --test runs it directly.
+
+// Keys a paged API commonly answers with. Read at the top level only: deeper
+// than that, a key with one of these names is somebody's data.
+const NEXT_CURSOR_KEYS = [
+  'nextCursor',
+  'next_cursor',
+  'nextPageToken',
+  'next_page_token',
+  'nextPage',
+  'next_page',
+  'next'
+];
+
+/**
+ * The cursor for the next page a JSON result names at its top level, if any.
+ * Cut text can lose it, so it is read from the parsed value before cutting.
+ */
+export const findNextCursor = (value: unknown): string | undefined => {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return undefined;
+  }
+  for (const key of NEXT_CURSOR_KEYS) {
+    const found = (value as Record<string, unknown>)[key];
+    if (typeof found === 'string' && found) return `${key}: ${found}`;
+    if (typeof found === 'number') return `${key}: ${found}`;
+  }
+  return undefined;
+};
+
+const kb = (bytes: number) => `${Math.max(1, Math.round(bytes / 1024))} KB`;
+
+/**
+ * `text` within `maxBytes` of UTF-8, or its start with a note that says it was
+ * cut. A character split by the cut is dropped rather than shown broken.
+ * `next` names the result's own cursor, for "call again with it".
+ */
+export const capText = (
+  text: string,
+  maxBytes: number,
+  next?: string
+): string => {
+  const bytes = new TextEncoder().encode(text);
+  if (bytes.byteLength <= maxBytes) return text;
+
+  const kept = new TextDecoder()
+    .decode(bytes.slice(0, maxBytes))
+    .replace(/�+$/, '');
+  const continueWith = next
+    ? ` The result names its next page (${next}): call again with it to continue.`
+    : ' To see the rest, call again asking for less — narrower filters, a smaller page or fewer fields.';
+  return `${kept}\n\n[Truncated: showing the first ${kb(maxBytes)} of ${kb(
+    bytes.byteLength
+  )}.${continueWith}]`;
+};
+
+// Opaque to the model, so it is passed back as given rather than edited.
+const encodeCursor = (key: string): string =>
+  btoa(String.fromCharCode(...new TextEncoder().encode(key)))
+    .replace(/\+/g, '-')
+    .replace(/\//g, '_')
+    .replace(/=+$/, '');
+
+const decodeCursor = (cursor: string): string | null => {
+  try {
+    const base64 = cursor.replace(/-/g, '+').replace(/_/g, '/');
+    const binary = atob(base64);
+    return new TextDecoder('utf-8', { fatal: true }).decode(
+      Uint8Array.from(binary, char => char.charCodeAt(0))
+    );
+  } catch {
+    return null;
+  }
+};
+
+// Between a cursor's key and the count of items with that key already served.
+// A key is a uri or a name, never a NUL.
+const CURSOR_SEEN_SEPARATOR = '\u0000';
+
+// A cursor with no count (one issued before the count was added) resumes
+// after every item with its key.
+const splitCursorKey = (decoded: string): { key: string; seen: number } => {
+  const at = decoded.lastIndexOf(CURSOR_SEEN_SEPARATOR);
+  if (at === -1) return { key: decoded, seen: Infinity };
+  const count = decoded.slice(at + 1);
+  if (!/^\d+$/.test(count)) throw new InvalidCursorError();
+  return { key: decoded.slice(0, at), seen: Number(count) };
+};
+
+export class InvalidCursorError extends Error {
+  constructor() {
+    super(
+      'That cursor is not valid. Call again without a cursor to start from the first page.'
+    );
+    this.name = 'InvalidCursorError';
+  }
+}
+
+/**
+ * What `list-resources` answers, wherever it runs (the MCP worker, and the
+ * channel runner, which lists proxied resources too): one page by uri, how many
+ * there are in all, and the cursor for the next page when there is one.
+ */
+export const listResourcesPage = (
+  resources: Array<{
+    uri: string;
+    title?: string | null;
+    description?: string | null;
+    mimeType?: string | null;
+  }>,
+  args: { cursor?: unknown; limit?: unknown },
+  limits: { defaultLimit: number; maxLimit: number }
+): string => {
+  const { page, total, nextCursor } = paginate(resources, r => r.uri, {
+    cursor: typeof args.cursor === 'string' ? args.cursor : undefined,
+    limit: typeof args.limit === 'number' ? args.limit : undefined,
+    ...limits
+  });
+  return JSON.stringify({
+    resources: page.map(r => ({
+      uri: r.uri,
+      title: r.title || undefined,
+      description: r.description || undefined,
+      mimeType: r.mimeType || undefined
+    })),
+    total,
+    ...(nextCursor ? { nextCursor } : {})
+  });
+};
+
+/**
+ * One page of `items`, ordered by `keyOf`, starting after the item the cursor
+ * names. Keyed rather than counted, so an item added or removed between calls
+ * doesn't shift the next page onto ones already seen.
+ */
+export const paginate = <T>(
+  items: T[],
+  keyOf: (item: T) => string,
+  options: {
+    cursor?: string;
+    limit?: number;
+    defaultLimit: number;
+    maxLimit: number;
+  }
+): { page: T[]; nextCursor?: string; total: number } => {
+  const limit = Math.min(
+    Math.max(Math.floor(Number(options.limit) || options.defaultLimit), 1),
+    options.maxLimit
+  );
+  const sorted = [...items].sort((a, b) => {
+    const left = keyOf(a);
+    const right = keyOf(b);
+    return left < right ? -1 : left > right ? 1 : 0;
+  });
+
+  let start = 0;
+  if (options.cursor) {
+    const decoded = decodeCursor(options.cursor);
+    if (decoded === null) throw new InvalidCursorError();
+    const { key: after, seen } = splitCursorKey(decoded);
+    start = sorted.findIndex(item => keyOf(item) >= after);
+    if (start === -1) start = sorted.length;
+    // Items that share the cursor's key: skip the ones already served, in the
+    // order the sort (which is stable) keeps them.
+    for (
+      let skipped = 0;
+      start < sorted.length && keyOf(sorted[start]) === after && skipped < seen;
+      skipped++
+    ) {
+      start++;
+    }
+  }
+
+  const page = sorted.slice(start, start + limit);
+  const more = start + limit < sorted.length;
+  if (!more || page.length === 0) return { page, total: sorted.length };
+
+  // The last key served, and how many items with that key have been served
+  // so far, so a run of equal keys split across pages resumes inside the run.
+  const end = start + page.length;
+  const lastKey = keyOf(page[page.length - 1]);
+  let seen = 0;
+  for (let i = end - 1; i >= 0 && keyOf(sorted[i]) === lastKey; i--) seen++;
+  return {
+    page,
+    total: sorted.length,
+    nextCursor: encodeCursor(`${lastKey}${CURSOR_SEEN_SEPARATOR}${seen}`)
+  };
+};

package/src/processQueueBatch.ts

@@ -1,8 +1,12 @@
 import { isRateLimitError, isTransientInfraError } from './retry';
 import { constants } from './constants';
+import { isPermanentJobError, planQueueFailure } from './queueFailure';
+import type { QueueFailureKind } from './queueFailure';
 
 export interface QueueMessageLike<T> {
   body: T;
+  // Deliveries so far, including this one (Cloudflare Queues' `attempts`).
+  attempts?: number;
   ack: () => void;
   retry: (opts?: { delaySeconds?: number }) => void;
 }
@@ -12,32 +16,58 @@ export interface QueueBatchLike<T> {
   messages: readonly QueueMessageLike<T>[];
 }
 
+export interface QueueFailureContext {
+  // No delivery follows this one — the moment to mark the work failed. Before
+  // it, a retry may still succeed, and a status flipped to failed in between
+  // would show a resource as broken while it is being fixed.
+  final: boolean;
+}
+
 export interface ProcessQueueBatchHandlers<T> {
   process: (body: T) => Promise<void>;
-  onError: (error: unknown, body: T, queueName: string) => Promise<void>;
+  onError: (
+    error: unknown,
+    body: T,
+    queueName: string,
+    context: QueueFailureContext
+  ) => Promise<void>;
+  // The consumer's configured `max_retries`.
+  maxRetries?: number;
 }
 
+const failureKind = (error: unknown): QueueFailureKind => {
+  if (isPermanentJobError(error)) return 'permanent';
+  if (isTransientInfraError(error)) return 'transient';
+  if (isRateLimitError(error)) return 'rateLimit';
+  return 'other';
+};
+
 export const processQueueBatch = async <T>(
   batch: QueueBatchLike<T>,
   handlers: ProcessQueueBatchHandlers<T>
 ): Promise<void> => {
+  const maxRetries = handlers.maxRetries ?? constants.QUEUE_MAX_RETRIES;
   for (const message of batch.messages) {
     try {
       await handlers.process(message.body);
       message.ack();
     } catch (error) {
-      if (isTransientInfraError(error)) {
-        // Transient Cloudflare platform hiccup (e.g. a Durable Object /
-        // Container reset). The job itself is fine, so re-run it after a short
-        // backoff without reporting an error or marking the work failed.
-        message.retry({ delaySeconds: constants.TRANSIENT_BACKOFF_SECONDS });
-        continue;
-      }
-      await handlers.onError(error, message.body, batch.queue);
-      if (isRateLimitError(error)) {
-        message.retry({
-          delaySeconds: constants.RATE_LIMIT_BACKOFF_SECONDS
+      const plan = planQueueFailure(
+        failureKind(error),
+        message.attempts,
+        maxRetries
+      );
+      if (plan.report) {
+        await handlers.onError(error, message.body, batch.queue, {
+          final: plan.final
         });
+      }
+      if (plan.action === 'ack') {
+        message.ack();
+      } else if (plan.backoff === 'transient') {
+        message.retry({ delaySeconds: constants.TRANSIENT_BACKOFF_SECONDS });
+      } else if (plan.backoff === 'rateLimit') {
+        message.retry({ delaySeconds: constants.RATE_LIMIT_BACKOFF_SECONDS });
       } else {
         message.retry();
       }

package/src/queueFailure.ts

@@ -0,0 +1,63 @@
+// How a failed queue job is handled: retried, given up on, or reported. Kept
+// apart from processQueueBatch, which applies it, so the decision is testable
+// on its own.
+
+// A failure that no retry can fix — a credential that needs the user to
+// reconnect, a resource that no longer exists. The job is reported once and
+// acknowledged rather than retried into the dead-letter queue. Recognised by
+// its `permanent` flag rather than its class, so an error built by another
+// copy of this module still counts.
+export class PermanentJobError extends Error {
+  readonly permanent = true;
+
+  constructor(message: string) {
+    super(message);
+    this.name = 'PermanentJobError';
+  }
+}
+
+export const isPermanentJobError = (error: unknown): boolean =>
+  !!error &&
+  typeof error === 'object' &&
+  (error as { permanent?: unknown }).permanent === true;
+
+export type QueueFailureKind =
+  | 'permanent'
+  | 'transient'
+  | 'rateLimit'
+  | 'other';
+
+export interface QueueFailurePlan {
+  // Hand the failure to the job's onError, which reports it.
+  report: boolean;
+  // No delivery follows this one: the job's work may be marked failed.
+  final: boolean;
+  // What to tell the queue about this message.
+  action: 'ack' | 'retry';
+  // Which backoff a retry waits out; null for the queue's default.
+  backoff: 'transient' | 'rateLimit' | null;
+}
+
+// `attempts` is the queue's own count, 1 on the first delivery; a consumer
+// with `maxRetries` retries gets maxRetries + 1 deliveries in all.
+export const planQueueFailure = (
+  kind: QueueFailureKind,
+  attempts: number | undefined,
+  maxRetries: number
+): QueueFailurePlan => {
+  if (kind === 'permanent') {
+    return { report: true, final: true, action: 'ack', backoff: null };
+  }
+  const final = (attempts ?? 1) > maxRetries;
+  if (kind === 'transient') {
+    // A platform hiccup is retried quietly; only the last one is worth a
+    // report, since nothing will run the job after it.
+    return { report: final, final, action: 'retry', backoff: 'transient' };
+  }
+  return {
+    report: true,
+    final,
+    action: 'retry',
+    backoff: kind === 'rateLimit' ? 'rateLimit' : null
+  };
+};

package/src/resourceRerank.ts

@@ -15,7 +15,8 @@ type RerankResponse = {
 };
 
 /**
- * The cross-encoder second stage for resource search, or undefined when the
+ * The cross-encoder second stage for resource search, or undefined when it is
+ * off (RESOURCE_SEARCH_RERANK_ENABLED, unless `enabled` overrides it) or the
  * Worker has no `AI` binding — local dev without Workers AI keeps plain fusion.
  *
  * Returns one score per document, in input order. A response that doesn't
@@ -23,10 +24,12 @@ type RerankResponse = {
  * ranking would silently drop the chunks it left out.
  */
 export const createResourceReranker = (
-  source: EnvSource
+  source: EnvSource,
+  enabled: boolean = constants.RESOURCE_SEARCH_RERANK_ENABLED
 ):
   | ((query: string, documents: string[]) => Promise<number[] | null>)
   | undefined => {
+  if (!enabled) return undefined;
   const ai = (source.env as { AI?: RerankAiBinding } | undefined)?.AI;
   if (!ai) return undefined;
 
@@ -37,7 +40,11 @@ export const createResourceReranker = (
       top_k: documents.length
     })) as RerankResponse;
 
-    const scores: Array<number | undefined> = new Array(documents.length);
+    // Filled, not just sized: `some` skips the holes of a bare new Array(n), so
+    // a document the response left out would pass the check below unscored.
+    const scores: Array<number | undefined> = new Array(documents.length).fill(
+      undefined
+    );
     for (const item of result?.response ?? []) {
       if (
         typeof item.id === 'number' &&