mixdog 1.2.0 → 1.2.2

package/src/runtime/agent/orchestrator/providers/cursor-wire-stream-bridge.mjs

@@ -3,11 +3,30 @@
  * response: frame parsing, tool-batch hand-off to the run registry, the
  * close verdict and the bounded in-place restart.
  */
-import { isRetryableCursorStreamError } from './cursor-wire-guards.mjs';
-import { markProviderRecoveryExhausted } from './retry-classifier.mjs';
+import {
+  isCursorAuthError,
+  isCursorThrottleError,
+  isRetryableCursorStreamError,
+  withCursorFailureHint,
+} from './cursor-wire-guards.mjs';
+import { emitProviderRetryStage, markProviderRecoveryExhausted, retryAfterMsFromError } from './retry-classifier.mjs';
 import { createFrameParser, cursorError, parseEndStream } from './cursor-wire-transport.mjs';
 import { processServerMessage } from './cursor-wire-stream-messages.mjs';
 
+// Resolves after `ms`, or early when the stream is cancelled.
+function abortableSleep(ms, signal) {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const done = () => {
+      clearTimeout(timer);
+      signal?.removeEventListener('abort', done);
+      resolve();
+    };
+    const timer = setTimeout(done, ms);
+    signal?.addEventListener('abort', done, { once: true });
+  });
+}
+
 /**
  * @param {object} ctx
  * @param {{ bridge, heartbeat }} ctx.live  the bridge currently feeding the stream (mutated on restart)
@@ -16,7 +35,10 @@ import { processServerMessage } from './cursor-wire-stream-messages.mjs';
  */
 export function createBridgeController(ctx) {
   const { live, state, filter, watchdog, sink, tuning, run, runs, restart, isCancelled } = ctx;
+  const { sleep = abortableSleep, cancelSignal = null, onStageChange = null } = ctx;
   let retryCount = 0;
+  let throttleCount = 0;
+  let authCount = 0;
 
   const finishToolBatch = () => {
     if (state.closed || state.pending.length === 0) return;
@@ -35,37 +57,91 @@ export function createBridgeController(ctx) {
     sink.finish('tool_calls');
   };
 
+  // Exponential backoff with +-20% jitter for in-place retries.
+  const backoffMs = (attempt) => {
+    const ceiling = Math.min(tuning.retryMaxDelayMs, tuning.retryBaseDelayMs * 2 ** (attempt - 1));
+    return Math.floor(ceiling * (0.8 + Math.random() * 0.4));
+  };
+
   const recoverOrFail = (error) => {
     const retryable = isRetryableCursorStreamError(error);
+    // A throttle retries within the bounded throttle retry count; otherwise
+    // the error surfaces with retryAfterMs for the outer layer.
+    const throttled = !retryable && isCursorThrottleError(error);
+    // Rejected credentials get a forced token refresh before the retry.
+    const authFailure = !retryable && !throttled && isCursorAuthError(error);
+    // A throttle retry rotates the conversation and drops the checkpoint
+    // so unlike other recoveries it can only replay a run that
+    // has shown nothing; the rest may also continue from a checkpoint.
     const recoveryEligible =
-      !isCancelled() && typeof restart === 'function' && (!state.visibleOutput || run.conversation.checkpoint);
-    const canRetry = retryCount < tuning.maxRetries && recoveryEligible && retryable;
+      !isCancelled() &&
+      typeof restart === 'function' &&
+      (!state.visibleOutput || (!throttled && Boolean(run.conversation.checkpoint)));
+    const serverDelay = retryAfterMsFromError(error);
+    // Server advice is honored in full for every retryable kind; local
+    // backoff applies only when the server gave none.
+    const delayMs = Number.isFinite(serverDelay)
+      ? serverDelay
+      : backoffMs(throttled ? throttleCount + 1 : retryCount + 1);
+    const throttleOk = throttled && throttleCount < tuning.maxThrottleRetries;
+    const authOk = authFailure && authCount < tuning.maxAuthRetries;
+    const canRetry =
+      retryCount < tuning.maxRetries && recoveryEligible && (retryable || throttleOk || authOk);
     if (!canRetry) {
       // A fresh Cursor run owns its bounded in-place retries. Mark
       // exhaustion so the outer loop does not multiply that budget.
       // Resumed tool-result streams have no restart closure; those
       // intentionally fall through unmarked so the outer loop can
       // rebuild from the committed assistant/tool-result history.
-      if (retryable && recoveryEligible && retryCount >= tuning.maxRetries) {
+      // A throttle that spent its bounded retries is final for the turn too.
+      if (
+        recoveryEligible &&
+        ((retryable && retryCount >= tuning.maxRetries) ||
+          (throttled && throttleCount >= tuning.maxThrottleRetries))
+      ) {
         markProviderRecoveryExhausted(error, { owner: 'cursor-wire', attempts: retryCount + 1 });
       }
-      sink.fail(error);
+      sink.fail(withCursorFailureHint(error));
       return;
     }
     retryCount += 1;
+    if (throttled) throttleCount += 1;
+    if (authFailure) authCount += 1;
     state.sawEnd = false;
     state.batchBoundaryReady = false;
     state.batchBoundaryChunkSeq = -1;
-    try {
-      const next = restart({
-        attempt: retryCount,
-        fromCheckpoint: Boolean(run.conversation.checkpoint),
-        visibleOutput: state.visibleOutput,
-      });
-      attach(next.bridge, next.heartbeat);
-    } catch (restartError) {
-      sink.fail(restartError);
-    }
+    // The wait is intentional, not stream silence; attach() re-arms on restart.
+    watchdog.pause();
+    emitProviderRetryStage(onStageChange, {
+      attempt: retryCount + 1,
+      maxAttempts: tuning.maxRetries + 1,
+      lastErr: error,
+      delayMs,
+    });
+    void sleep(delayMs, cancelSignal).then(async () => {
+      if (isCancelled() || state.closed) return;
+      try {
+        // restart() may refresh credentials first, so it can be async.
+        const next = await restart({
+          attempt: retryCount,
+          fromCheckpoint: !throttled && Boolean(run.conversation.checkpoint),
+          visibleOutput: state.visibleOutput,
+          reason: throttled ? 'throttle' : authFailure ? 'auth' : 'transient',
+          error,
+        });
+        if (isCancelled() || state.closed) {
+          clearInterval(next.heartbeat);
+          next.bridge.close();
+          return;
+        }
+        attach(next.bridge, next.heartbeat);
+        try {
+          onStageChange?.('streaming');
+        } catch {}
+      } catch (restartError) {
+        sink.fail(withCursorFailureHint(restartError));
+      }
+    });
   };
 
   /** The Connect end frame: an error closes, pending tools delimit, a finished turn completes. */
@@ -99,20 +175,23 @@ export function createBridgeController(ctx) {
         closeError = frameError;
       }
     }
-    // Cursor commonly closes HTTP/2 immediately after turnEnded without
-    // a separate Connect end frame. The turn is already complete.
-    if (state.sawTurnEnded) {
-      sink.finish();
-      return;
-    }
     // Tool calls already emitted to the caller remain actionable even if
     // the parked transport vanished. The next request rebuilds/resumes.
+    // Checked before turnEnded: a close that follows turnEnded only ends a
+    // completed turn when no exec is still waiting (turn ended
+    // with no MCP exec outstanding), never by dropping a tool call.
     if (state.pending.length > 0) {
       sink.finish('tool_calls');
       return;
     }
+    // Cursor commonly closes HTTP/2 immediately after turnEnded, with or
+    // without an end-stream error. The turn is already complete.
+    if (state.sawTurnEnded) {
+      sink.finish();
+      return;
+    }
     if (!closeError && !state.sawEnd) {
-      closeError = cursorError('Cursor stream closed before its end frame', { code: 'protocol_error' });
+      closeError = cursorError('Cursor stream closed before its end frame', { code: 'incomplete_stream' });
     }
     if (closeError) recoverOrFail(closeError);
     else sink.finish();

package/src/runtime/agent/orchestrator/providers/cursor-wire-transport.mjs

@@ -4,6 +4,7 @@ import crypto from 'node:crypto';
 import http2 from 'node:http2';
 import { MAX_CONNECT_FRAME_BYTES, createCursorByteQueue } from './cursor-wire-guards.mjs';
 import { cursorClientVersion, warmCursorClientVersion } from './cursor-client-version.mjs';
+import { retryAfterMsFromError } from './retry-classifier.mjs';
 
 export { warmCursorClientVersion };
 
@@ -40,7 +41,66 @@ function rpcHeaders(accessToken, path, unary) {
   };
 }
 
-export function cursorError(message, { status = 0, code = '', retryAfter = null } = {}) {
+// HTTP-equivalent status for every Connect error code.
+const CONNECT_CODE_STATUS = Object.freeze({
+  canceled: 499,
+  unknown: 500,
+  invalid_argument: 400,
+  deadline_exceeded: 504,
+  not_found: 404,
+  already_exists: 409,
+  permission_denied: 403,
+  resource_exhausted: 429,
+  failed_precondition: 400,
+  aborted: 409,
+  out_of_range: 400,
+  unimplemented: 501,
+  internal: 500,
+  unavailable: 503,
+  data_loss: 500,
+  unauthenticated: 401,
+});
+
+// Connect code for a plain HTTP failure that carried no Connect error body.
+const HTTP_STATUS_CODE = Object.freeze({
+  401: 'unauthenticated',
+  403: 'permission_denied',
+  404: 'not_found',
+  429: 'resource_exhausted',
+  500: 'internal',
+  502: 'unavailable',
+  503: 'unavailable',
+  504: 'deadline_exceeded',
+});
+
+// Only explicit wording about the tool catalog or request payload being too
+// large counts; resource_exhausted otherwise stays a throttle.
+const REQUEST_TOO_LARGE_RE =
+  /\b(?:tool\s+(?:catalog|definitions?|schemas?)|tools|payload|request(?:\s+body)?)\b[^.\n]{0,80}?\b(?:too\s+(?:large|big)|exceed(?:s|ed)?\b[^.\n]{0,30}?\b(?:size|limit|maximum))/i;
+
+// Rate/quota wording wins over the size match: "request exceeded the rate
+// limit" is a throttle, not an oversized request.
+const THROTTLE_CUE_RE = /too\s+many\s+requests|quota|rate[\s_-]*limit|throttl/i;
+
+function firstValue(value) {
+  return Array.isArray(value) ? value[0] : value;
+}
+
+// Lowercased string-valued header map without HTTP/2 pseudo-headers.
+function plainHeaders(source) {
+  const headers = {};
+  for (const [key, value] of Object.entries(source || {})) {
+    const first = firstValue(value);
+    if (key.startsWith(':') || first == null || typeof first === 'object') continue;
+    headers[key.toLowerCase()] = String(first);
+  }
+  return headers;
+}
+
+export function cursorError(
+  message,
+  { status = 0, code = '', cursorCode = code, headers = null, retryAfter = null, reauthRequired = false, detail } = {}
+) {
   const error = new Error(message);
   if (status) {
     error.status = status;
@@ -48,17 +108,32 @@ export function cursorError(message, { status = 0, code = '', retryAfter = null
   }
   if (code) {
     error.code = code;
-    error.cursorCode = code;
+    error.cursorCode = cursorCode;
   }
+  if (detail != null) error.cursorDetail = detail;
+  if (reauthRequired) error.reauthRequired = true;
+  const errorHeaders = plainHeaders(headers);
   if (retryAfter != null && retryAfter !== '') {
-    const headers = { 'retry-after': String(retryAfter) };
     error.retryAfter = retryAfter;
-    error.headers = headers;
-    error.response = { status, headers };
+    errorHeaders['retry-after'] = String(retryAfter);
+  }
+  if (Object.keys(errorHeaders).length) {
+    error.headers = errorHeaders;
+    error.response = { status, headers: errorHeaders };
+    const retryAfterMs = retryAfterMsFromError(error);
+    if (Number.isFinite(retryAfterMs)) error.retryAfterMs = retryAfterMs;
   }
   return error;
 }
 
+function httpStatusError(status, headers) {
+  return cursorError(`Cursor request failed (${status})`, {
+    status,
+    code: HTTP_STATUS_CODE[status] || '',
+    headers,
+  });
+}
+
 export function openCursorStream({ accessToken, path = RUN_PATH, url = API_URL }) {
   const session = http2.connect(url);
   const request = session.request(rpcHeaders(accessToken, path, false));
@@ -66,7 +141,7 @@ export function openCursorStream({ accessToken, path = RUN_PATH, url = API_URL }
   let closeHandler = null;
   let closed = false;
   let status = 0;
-  let retryAfter = null;
+  let responseHeaders = null;
   let closeError = null;
   let timeout = setTimeout(
     () => close(cursorError('Cursor connection timed out', { code: 'connection_timeout' })),
@@ -97,8 +172,7 @@ export function openCursorStream({ accessToken, path = RUN_PATH, url = API_URL }
     closed = true;
     clearTimeout(timeout);
     clearInterval(ping);
-    closeError =
-      error || (status >= 400 ? cursorError(`Cursor request failed (${status})`, { status, retryAfter }) : null);
+    closeError = error || (status >= 400 ? httpStatusError(status, responseHeaders) : null);
     try {
       request.close();
     } catch {}
@@ -120,7 +194,7 @@ export function openCursorStream({ accessToken, path = RUN_PATH, url = API_URL }
 
   request.on('response', (headers) => {
     status = Number(headers[':status'] || 0);
-    retryAfter = headers['retry-after'] ?? null;
+    responseHeaders = headers;
     resetTimeout();
   });
   request.on('data', (chunk) => {
@@ -167,7 +241,7 @@ export async function callCursorUnary({ accessToken, path, body, url = API_URL,
     const request = session.request(rpcHeaders(accessToken, path, true));
     const chunks = [];
     let status = 0;
-    let retryAfter = null;
+    let responseHeaders = null;
     let settled = false;
     const timeout = setTimeout(() => finish(new Error('Cursor request timed out')), timeoutMs);
     const finish = (error = null) => {
@@ -181,12 +255,12 @@ export async function callCursorUnary({ accessToken, path, body, url = API_URL,
         session.close();
       } catch {}
       if (error) reject(error);
-      else if (status >= 400) reject(cursorError(`Cursor request failed (${status})`, { status, retryAfter }));
+      else if (status >= 400) reject(httpStatusError(status, responseHeaders));
       else resolve(Buffer.concat(chunks));
     };
     request.on('response', (headers) => {
       status = Number(headers[':status'] || 0);
-      retryAfter = headers['retry-after'] ?? null;
+      responseHeaders = headers;
     });
     request.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
     request.on('end', () => finish());
@@ -235,7 +309,7 @@ export function createFrameParser(onMessage, onEnd) {
   };
   parse.finish = () => {
     if (pending.byteLength) {
-      throw cursorError('Cursor stream ended with a truncated frame', { code: 'protocol_error' });
+      throw cursorError('Cursor stream ended with a truncated frame', { code: 'incomplete_stream' });
     }
   };
   parse.bufferedBytes = () => pending.byteLength;
@@ -246,26 +320,22 @@ export function parseEndStream(bytes) {
   try {
     const payload = JSON.parse(textDecoder.decode(bytes));
     if (!payload?.error) return null;
-    const code = String(payload.error.code || 'error');
-    const status =
-      {
-        unauthenticated: 401,
-        permission_denied: 403,
-        not_found: 404,
-        resource_exhausted: 429,
-        invalid_argument: 400,
-        internal: 500,
-        unavailable: 503,
-      }[code] || 0;
-    return cursorError(`Cursor ${code}: ${payload.error.message || 'request failed'}`, {
-      status,
-      code,
-      retryAfter:
-        payload.error.retryAfter ??
-        payload.error.retry_after ??
-        payload.metadata?.['retry-after'] ??
-        payload.metadata?.retryAfter ??
-        null,
+    const connectCode = String(payload.error.code || 'error').toLowerCase();
+    const detail = String(payload.error.message || 'request failed');
+    const tooLarge = connectCode === 'resource_exhausted' && !THROTTLE_CUE_RE.test(detail) && REQUEST_TOO_LARGE_RE.test(detail);
+    const retryAfter =
+      payload.error.retryAfter ??
+      payload.error.retry_after ??
+      payload.metadata?.['retry-after'] ??
+      payload.metadata?.retryAfter ??
+      null;
+    return cursorError(`Cursor ${connectCode}: ${detail}`, {
+      status: tooLarge ? 413 : CONNECT_CODE_STATUS[connectCode] || 0,
+      code: tooLarge ? 'request_too_large' : connectCode,
+      cursorCode: connectCode,
+      headers: payload.metadata,
+      retryAfter: firstValue(retryAfter),
+      detail,
     });
   } catch {
     return cursorError('Cursor returned an invalid end-stream frame', { code: 'protocol_error' });

package/src/runtime/agent/orchestrator/providers/cursor-wire.mjs

@@ -14,6 +14,7 @@ import {
 } from './cursor-wire-protobuf.mjs';
 import {
   assertCursorUserImages,
+  createCursorCredentialRefresher,
   createCursorStreamWatchdog,
   resolveCursorStreamTuning,
   storeCursorBlob,
@@ -358,12 +359,16 @@ function createStreamResponse({
   sessionId = '',
   sawTurnEnded = false,
   restart = null,
+  sleep = undefined,
+  onStageChange = null,
 }) {
   const id = newCompletionId();
   // The bridge currently feeding the stream; a restart swaps it in place.
   const live = { bridge, heartbeat };
   let watchdog = null;
   let cancelled = false;
+  // Cuts a pending retry backoff short when the caller cancels.
+  const cancelController = new AbortController();
   const runs = { get: (runKey) => activeRuns.get(runKey), store: storeActiveRun, forget: forgetActiveRun };
   const stream = new ReadableStream({
     start(controller) {
@@ -395,12 +400,16 @@ function createStreamResponse({
         run: { key, conversation, tools, cloudRule, modelParameters, maxMode, sessionId },
         runs,
         restart,
+        sleep,
+        onStageChange,
+        cancelSignal: cancelController.signal,
         isCancelled: () => cancelled,
       });
       bridges.attach(bridge, heartbeat);
     },
     cancel(reason) {
       cancelled = true;
+      cancelController.abort();
       watchdog?.stop();
       clearInterval(live.heartbeat);
       const active = activeRuns.get(key);
@@ -459,7 +468,7 @@ function noUserMessageResponse() {
   });
 }
 
-export async function handleChatCompletion(body, accessToken) {
+export async function handleChatCompletion(body, accessToken, { onStageChange = null, getAccessToken = null } = {}) {
   await warmCursorClientVersion();
   const parsed = parseMessages(body.messages);
   if (
@@ -504,6 +513,10 @@ export async function handleChatCompletion(body, accessToken) {
   };
   const requestBytes = buildRunRequest(runInput);
   const { bridge, heartbeat } = startRun(accessToken, requestBytes);
+  // Every retry force-refreshes credentials and falls back to the previous
+  // token if the refresh fails. A rejected credential is only
+  // retried when the refresh produced a different token.
+  const credentials = createCursorCredentialRefresher(accessToken, getAccessToken);
   return createStreamResponse({
     bridge,
     heartbeat,
@@ -515,11 +528,21 @@ export async function handleChatCompletion(body, accessToken) {
     maxMode,
     key,
     sessionId,
-    restart: ({ fromCheckpoint }) =>
-      startRun(
-        accessToken,
-        fromCheckpoint ? buildRunRequest({ ...runInput, userText: '', userImages: [] }) : requestBytes
-      ),
+    onStageChange,
+    restart: async ({ fromCheckpoint, reason, error }) => {
+      if (reason === 'throttle') {
+        // Cursor can pin resource_exhausted to one conversation id (conversation
+        // rotation after a rate limit): start a fresh one from the full
+        // history instead of the checkpoint it refused.
+        conversation.id = crypto.randomUUID();
+        conversation.checkpoint = null;
+      }
+      const token = await credentials.refresh(reason, error);
+      let bytes = requestBytes;
+      if (fromCheckpoint) bytes = buildRunRequest({ ...runInput, userText: '', userImages: [] });
+      else if (reason === 'throttle') bytes = buildRunRequest(runInput);
+      return startRun(token, bytes);
+    },
   });
 }
 

package/src/runtime/agent/orchestrator/providers/cursor.mjs

@@ -10,6 +10,7 @@ import {
 } from './openai-compat-wire.mjs';
 import { preparePdfTextForProvider } from './media-normalization.mjs';
 import { consumeCompatChatCompletionStream } from './openai-compat-stream.mjs';
+import { retryAfterMsFromError } from './retry-classifier.mjs';
 import { ensureChatToolPairs } from './lib/wire-pairing.mjs';
 import { cursorTokenExpiry, exchangeCursorToken, resolveCursorOAuthAccessToken } from './cursor-auth.mjs';
 import {
@@ -62,14 +63,24 @@ function toCursorToolChoice(toolChoice) {
 async function responseError(response, label) {
   const text = await response.text().catch(() => '');
   let message = text;
+  let code = '';
   try {
     const parsed = JSON.parse(text);
     message = parsed?.error?.message || parsed?.message || text;
+    code = String(parsed?.error?.code || parsed?.code || '').toLowerCase();
   } catch {
     /* plain text */
   }
   const error = new Error(`${label} request failed (${response.status})${message ? `: ${message}` : ''}`);
   error.status = response.status;
+  error.httpStatus = response.status;
+  if (code) error.code = code;
+  const headers = Object.fromEntries(response.headers?.entries?.() ?? []);
+  if (Object.keys(headers).length) {
+    error.headers = headers;
+    const retryAfterMs = retryAfterMsFromError(error);
+    if (Number.isFinite(retryAfterMs)) error.retryAfterMs = retryAfterMs;
+  }
   throw error;
 }
 
@@ -535,6 +546,9 @@ class CursorProviderBase {
   nativePdf = false;
 
   static inputExcludesCache = false;
+  // The in-place stream retry budget is the whole recovery: a spent budget
+  // surfaces instead of being re-run by the outer recovery cycles.
+  retryPolicy = Object.freeze({ recoveryCycles: false });
   name;
   config;
   _runtimeOverride;
@@ -622,7 +636,10 @@ class CursorProviderBase {
       sendOpts.onStageChange?.('requesting');
     } catch {}
     try {
-      const response = await runtime.handleChatCompletion(body, accessToken);
+      const response = await runtime.handleChatCompletion(body, accessToken, {
+        onStageChange: sendOpts.onStageChange,
+        getAccessToken: (options) => this._accessToken({ signal, ...options }),
+      });
       try {
         sendOpts.onStageChange?.('streaming');
       } catch {}

package/src/runtime/agent/orchestrator/providers/custom-provider.mjs

@@ -4,6 +4,7 @@
 // runtime reuses the existing OpenAI-compat / Anthropic engines, constructed
 // under the custom id so model routes and provider identity stay `custom-…`.
 
+import { resolveProviderApiKey } from '../../../shared/provider-api-key.mjs';
 import { effectiveCustomModelRows, normalizeCustomProviderConfig } from './custom-provider-config.mjs';
 import { ANTHROPIC_VERSION } from './lib/anthropic-models.mjs';
 import { enrichModels } from './model-catalog.mjs';
@@ -39,8 +40,11 @@ function withCustomModels(Base, discover) {
     customConfig = null;
     customDiscoveryError = null;
     _customDiscovered = null;
-    _customApiKey = '';
     _customKeyless = false;
+    // The key for discovery and error redaction, resolved per use like sends.
+    _customKey() {
+      return this._customKeyless ? '' : resolveProviderApiKey(this.name, this.config);
+    }
     _customRows() {
       return effectiveCustomModelRows(this.customConfig, this._customDiscovered || [], { providerId: this.name });
     }
@@ -57,7 +61,7 @@ function withCustomModels(Base, discover) {
         this.customDiscoveryError = null;
       } catch (error) {
         if (signal?.aborted) throw error;
-        failure = sanitizeDiscoveryError(error, this._customApiKey);
+        failure = sanitizeDiscoveryError(error, this._customKey());
         this.customDiscoveryError = failure;
       }
       const rows = this._customRows();
@@ -82,7 +86,7 @@ async function discoverAnthropicModels(provider, signal) {
     method: 'GET',
     signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
     headers: {
-      ...(provider._customKeyless ? {} : { 'x-api-key': provider.apiKey || '' }),
+      ...(provider._customKeyless ? {} : { 'x-api-key': provider._customKey() }),
       'anthropic-version': ANTHROPIC_VERSION,
     },
   });
@@ -141,7 +145,6 @@ export async function createCustomProvider(id, config) {
     provider = new Custom(providerId, engine);
   }
   provider.customConfig = cfg;
-  provider._customApiKey = apiKey;
   provider._customKeyless = keyless;
   provider._applyCustomRows(provider._customRows());
   return provider;