@madewithremy/admin 0.1.7 → 0.1.8

package/dist/index.js

@@ -374,8 +374,11 @@ __export(dataSources_exports, {
   promote: () => promote,
   revectorize: () => revectorize,
   rm: () => rm,
+  rmWhere: () => rmWhere,
+  rmWhereAll: () => rmWhereAll,
   search: () => search,
   summarize: () => summarize,
+  waitForCandidate: () => waitForCandidate,
   waitForIngest: () => waitForIngest,
   waitForMove: () => waitForMove
 });
@@ -386,6 +389,48 @@ function sleep(ms) {
   return new Promise((r) => setTimeout(r, ms));
 }
 
+// src/poll.ts
+async function pollUntil(read, isSettled, options) {
+  const start2 = Date.now();
+  let transientSince = null;
+  while (true) {
+    let value;
+    try {
+      value = await read();
+      transientSince = null;
+    } catch (err) {
+      if (!isTransientError(err)) {
+        throw err;
+      }
+      transientSince ??= Date.now();
+      if (Date.now() - transientSince > TRANSIENT_GRACE_MS) {
+        throw err;
+      }
+      options.onProgress?.(
+        `retrying after a gateway error\u2026 (${elapsedSeconds(start2)}s)`
+      );
+      await sleep(options.pollMs);
+      continue;
+    }
+    if (isSettled(value)) {
+      return { settled: true, value };
+    }
+    if (Date.now() - start2 > options.timeoutMs) {
+      return { settled: false, value };
+    }
+    if (options.describe) {
+      options.onProgress?.(options.describe(value, start2));
+    }
+    await sleep(options.pollMs);
+  }
+}
+function elapsedSeconds(start2) {
+  return Math.round((Date.now() - start2) / 1e3);
+}
+function timedOut(timeoutMs, still) {
+  return `Timed out after ${Math.round(timeoutMs / 1e3)}s ${still}`;
+}
+
 // src/upload.ts
 async function uploadDirect(upload, bytes, filename) {
   const form = new FormData();
@@ -437,7 +482,12 @@ async function addDocument(ctx, params) {
   );
   if (token.alreadyCurrent) {
     onProgress?.(`${filename}: unchanged, skipped`);
-    return { filename, skipped: true, document: token.document };
+    return {
+      filename,
+      skipped: true,
+      document: token.document,
+      documents: token.documents ?? [token.document]
+    };
   }
   onProgress?.(
     `${filename}: uploading ${(content.length / 1024 / 1024).toFixed(1)}MB\u2026`
@@ -460,7 +510,9 @@ async function addDocument(ctx, params) {
     filename,
     skipped: false,
     queued: confirmed.queued,
-    document: confirmed.document
+    document: confirmed.document,
+    documents: confirmed.documents ?? (confirmed.document ? [confirmed.document] : []),
+    outcome: confirmed.outcome
   };
 }
 async function waitForIngest(ctx, params) {
@@ -470,50 +522,64 @@ async function waitForIngest(ctx, params) {
     return { status: "up-to-date", documents: [] };
   }
   const wanted = new Set(documentIds);
-  const start = Date.now();
-  let transientSince = null;
-  for (; ; ) {
-    let docs;
-    try {
-      docs = await allDocuments(ctx, { slug, ids: documentIds });
-      transientSince = null;
-    } catch (err) {
-      if (!isTransientError(err)) {
-        throw err;
-      }
-      transientSince ??= Date.now();
-      if (Date.now() - transientSince > TRANSIENT_GRACE_MS) {
-        throw err;
-      }
-      onProgress?.(
-        `retrying after a gateway error\u2026 (${Math.round((Date.now() - start) / 1e3)}s)`
-      );
-      await sleep(POLL_MS);
-      continue;
-    }
-    const tracked = (docs ?? []).filter((d) => wanted.has(d.id));
-    const pending = tracked.filter((d) => d.status === "processing");
-    const failed = tracked.filter((d) => d.status === "error");
-    if (pending.length === 0) {
-      return {
-        status: failed.length ? "error" : "done",
-        documents: tracked.map(summarize)
-      };
+  const pendingOf = (docs) => docs.filter((d) => d.status === "processing");
+  const outcome = await pollUntil(
+    async () => {
+      const docs = await allDocuments(ctx, { slug, ids: documentIds });
+      return (docs ?? []).filter((d) => wanted.has(d.id));
+    },
+    (tracked2) => pendingOf(tracked2).length === 0,
+    {
+      timeoutMs,
+      pollMs: POLL_MS,
+      describe: (tracked2, start2) => `ingesting\u2026 ${tracked2.length - pendingOf(tracked2).length}/${tracked2.length} done (${elapsedSeconds(start2)}s)`,
+      onProgress
     }
-    if (Date.now() - start > timeoutMs) {
-      return {
-        status: "timeout",
-        documents: tracked.map(summarize),
-        error: `Timed out after ${Math.round(timeoutMs / 1e3)}s with ${pending.length} document(s) still processing`
-      };
+  );
+  const tracked = outcome.value;
+  if (!outcome.settled) {
+    return {
+      status: "timeout",
+      documents: tracked.map(summarize),
+      error: timedOut(
+        timeoutMs,
+        `with ${pendingOf(tracked).length} document(s) still processing`
+      )
+    };
+  }
+  return {
+    status: tracked.some((d) => d.status === "error") ? "error" : "done",
+    documents: tracked.map(summarize)
+  };
+}
+async function waitForCandidate(ctx, params) {
+  const timeoutMs = params.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS;
+  const pendingOf = (docs2) => docs2.filter((d) => d.status === "processing");
+  const outcome = await pollUntil(
+    () => allDocuments(ctx, { slug: params.slug, candidate: true }),
+    (docs2) => pendingOf(docs2).length === 0,
+    {
+      timeoutMs,
+      pollMs: POLL_MS,
+      describe: (docs2, start2) => `rebuilding\u2026 ${docs2.length - pendingOf(docs2).length}/${docs2.length} done (${elapsedSeconds(start2)}s)`,
+      onProgress: params.onProgress
     }
-    onProgress?.(
-      `ingesting\u2026 ${tracked.length - pending.length}/${tracked.length} done (${Math.round(
-        (Date.now() - start) / 1e3
-      )}s)`
-    );
-    await sleep(POLL_MS);
+  );
+  const docs = outcome.value;
+  if (!outcome.settled) {
+    return {
+      status: "timeout",
+      documents: docs.map(summarize),
+      error: timedOut(
+        timeoutMs,
+        `with ${pendingOf(docs).length} document(s) still building`
+      )
+    };
   }
+  return {
+    status: docs.some((d) => d.status === "error") ? "error" : "ready",
+    documents: docs.map(summarize)
+  };
 }
 var DOCUMENTS_IDS_MAX = 200;
 function documents(ctx, params) {
@@ -621,52 +687,39 @@ function move(ctx, params) {
 }
 async function waitForMove(ctx, params) {
   const timeoutMs = params.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS;
-  const start = Date.now();
-  let transientSince = null;
-  for (; ; ) {
-    let source;
-    try {
+  const outcome = await pollUntil(
+    async () => {
       const { dataSources } = await list3(ctx);
-      source = dataSources.find((s) => s.slug === params.slug);
-      transientSince = null;
-    } catch (err) {
-      if (!isTransientError(err)) {
-        throw err;
-      }
-      transientSince ??= Date.now();
-      if (Date.now() - transientSince > TRANSIENT_GRACE_MS) {
-        throw err;
+      const source2 = dataSources.find((s) => s.slug === params.slug);
+      if (!source2) {
+        throw new Error(`Data source "${params.slug}" not found.`);
       }
-      params.onProgress?.(
-        `retrying after a gateway error\u2026 (${Math.round((Date.now() - start) / 1e3)}s)`
-      );
-      await sleep(POLL_MS);
-      continue;
-    }
-    if (!source) {
-      throw new Error(`Data source "${params.slug}" not found.`);
-    }
-    const migration = source.migration;
-    if (!migration) {
-      return { status: "done", source };
-    }
-    if (migration.error) {
-      return { status: "failed", source, error: migration.error };
-    }
-    if (Date.now() - start > timeoutMs) {
-      return {
-        status: "timeout",
-        source,
-        error: `Timed out after ${Math.round(timeoutMs / 1e3)}s with ${migration.copied}/${migration.total} documents copied`
-      };
+      return source2;
+    },
+    (source2) => !source2.migration || source2.migration.error !== null,
+    {
+      timeoutMs,
+      pollMs: POLL_MS,
+      describe: (source2, start2) => `moving\u2026 ${source2.migration.copied}/${source2.migration.total} documents (${elapsedSeconds(start2)}s)`,
+      onProgress: params.onProgress
     }
-    params.onProgress?.(
-      `moving\u2026 ${migration.copied}/${migration.total} documents (${Math.round(
-        (Date.now() - start) / 1e3
-      )}s)`
-    );
-    await sleep(POLL_MS);
+  );
+  const source = outcome.value;
+  const migration = source.migration;
+  if (!outcome.settled) {
+    return {
+      status: "timeout",
+      source,
+      error: timedOut(
+        timeoutMs,
+        `with ${migration.copied}/${migration.total} documents copied`
+      )
+    };
+  }
+  if (migration?.error) {
+    return { status: "failed", source, error: migration.error };
   }
+  return { status: "done", source };
 }
 function revectorize(ctx, params) {
   const { slug, ingest } = params;
@@ -713,6 +766,27 @@ function rm(ctx, params) {
     }
   );
 }
+function rmWhere(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base(ctx.appId)}/documents/delete-many`,
+    { slug: params.slug, filter: params.filter }
+  );
+}
+async function rmWhereAll(ctx, params) {
+  let deleted = 0;
+  while (true) {
+    const page = await rmWhere(ctx, params);
+    deleted += page.deleted;
+    if (page.remaining === 0 || page.deleted === 0) {
+      return { deleted };
+    }
+    params.onProgress?.(
+      `removed ${deleted.toLocaleString()} so far, ${page.remaining.toLocaleString()} remaining\u2026`
+    );
+  }
+}
 function deleteSource(ctx, slug) {
   return call(
     ctx,
@@ -722,6 +796,526 @@ function deleteSource(ctx, slug) {
   );
 }
 
+// src/ops/dataSourceJobs.ts
+var dataSourceJobs_exports = {};
+__export(dataSourceJobs_exports, {
+  DEFAULT_WAIT_TIMEOUT_MS: () => DEFAULT_WAIT_TIMEOUT_MS2,
+  approve: () => approve,
+  cancel: () => cancel,
+  describeProgress: () => describeProgress,
+  get: () => get2,
+  list: () => list4,
+  pause: () => pause,
+  resume: () => resume,
+  start: () => start,
+  uploadManifest: () => uploadManifest,
+  waitForJob: () => waitForJob,
+  waitForPlan: () => waitForPlan
+});
+var DEFAULT_WAIT_TIMEOUT_MS2 = 60 * 60 * 1e3;
+var POLL_MS2 = 5e3;
+function base2(appId) {
+  return `/_internal/v2/apps/${appId}/datasources/jobs`;
+}
+function start(ctx, params) {
+  const { slug, source, approve: approve2, budgetDollars, concurrency, limit } = params;
+  return call(ctx, "POST", base2(ctx.appId), {
+    slug,
+    source,
+    ...approve2 ? { approve: true } : {},
+    ...budgetDollars !== void 0 ? { budgetDollars } : {},
+    ...concurrency !== void 0 ? { concurrency } : {},
+    ...limit !== void 0 ? { limit } : {}
+  });
+}
+async function uploadManifest(ctx, params) {
+  const token = await call(
+    ctx,
+    "POST",
+    `${base2(ctx.appId)}/manifest-token`,
+    { slug: params.slug }
+  );
+  await uploadDirect(token.upload, params.content, "manifest.jsonl");
+  return { key: token.key };
+}
+function list4(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base2(ctx.appId)}${qs({ slug: params?.slug })}`
+  );
+}
+function get2(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}`
+  );
+}
+function approve(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/approve`
+  );
+}
+function pause(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/pause`
+  );
+}
+function resume(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/resume`
+  );
+}
+function cancel(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/cancel`
+  );
+}
+function waitForPlan(ctx, params) {
+  return settle(ctx, params, (job) => job.state !== "planning");
+}
+function waitForJob(ctx, params) {
+  return settle(
+    ctx,
+    params,
+    (job) => !["planning", "planned", "running"].includes(job.state) || job.state === "planned" && (job.error !== null || !job.autoApprove)
+  );
+}
+async function settle(ctx, params, isSettled) {
+  const timeoutMs = params.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS2;
+  const outcome = await pollUntil(
+    async () => (await get2(ctx, { id: params.id })).job,
+    isSettled,
+    {
+      timeoutMs,
+      pollMs: POLL_MS2,
+      describe: describeProgress,
+      onProgress: params.onProgress
+    }
+  );
+  const job = outcome.value;
+  if (!outcome.settled) {
+    return {
+      status: "timeout",
+      job,
+      error: timedOut(timeoutMs, `; job is still ${job.state}`)
+    };
+  }
+  if (job.state === "failed" || job.state === "cancelled") {
+    return {
+      status: "failed",
+      job,
+      error: job.error ?? `The job was ${job.state}.`
+    };
+  }
+  if (job.state === "paused") {
+    return {
+      status: "failed",
+      job,
+      error: job.error ?? `The job paused (${job.pauseReason ?? "user"}). Resume it to continue.`
+    };
+  }
+  if (job.state === "planned" && (job.error !== null || job.autoApprove)) {
+    return {
+      status: "planned",
+      job,
+      error: job.error ?? "The plan is ready but the job did not start. Approve it once the cause is fixed."
+    };
+  }
+  return { status: "done", job };
+}
+function describeProgress(job, start2) {
+  const elapsed = `(${elapsedSeconds(start2)}s)`;
+  if (job.state === "planning") {
+    return job.enumerationDone ? `planning\u2026 reading a sample of ${job.counts.objectsSeen.toLocaleString()} objects ${elapsed}` : `planning\u2026 ${job.counts.objectsSeen.toLocaleString()} objects counted ${elapsed}`;
+  }
+  const total = job.plan?.projected.documents ?? job.counts.objectsSeen;
+  const processed = job.counts.documentsDone + job.counts.documentsSkipped + job.counts.documentsFailed;
+  const spend = `$${job.estimatedDollars.toFixed(2)}${job.budgetDollars !== null ? ` of $${job.budgetDollars.toFixed(2)}` : ""}`;
+  return `${job.state}\u2026 ${processed.toLocaleString()}/${total.toLocaleString()} documents \xB7 ${spend} ${elapsed}`;
+}
+
+// src/ops/dataSourceConnectors.ts
+var dataSourceConnectors_exports = {};
+__export(dataSourceConnectors_exports, {
+  connect: () => connect,
+  disconnect: () => disconnect,
+  get: () => get3,
+  sync: () => sync
+});
+function base3(appId) {
+  return `/_internal/v2/apps/${appId}/datasources`;
+}
+function connect(ctx, params) {
+  const {
+    slug,
+    bucket,
+    region,
+    prefix,
+    endpoint,
+    accessKeyIdSecret,
+    secretAccessKeySecret,
+    deletions,
+    budgetDollarsPerSync
+  } = params;
+  return call(
+    ctx,
+    "POST",
+    `${base3(ctx.appId)}/connector`,
+    {
+      slug,
+      bucket,
+      region,
+      ...prefix !== void 0 ? { prefix } : {},
+      ...endpoint !== void 0 ? { endpoint } : {},
+      accessKeyIdSecret,
+      secretAccessKeySecret,
+      ...deletions !== void 0 ? { deletions } : {},
+      ...budgetDollarsPerSync !== void 0 ? { budgetDollarsPerSync } : {}
+    }
+  );
+}
+function get3(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base3(ctx.appId)}/connector${qs({ slug: params.slug })}`
+  );
+}
+function disconnect(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base3(ctx.appId)}/connector/remove`,
+    { slug: params.slug }
+  );
+}
+function sync(ctx, params) {
+  return call(ctx, "POST", `${base3(ctx.appId)}/sync`, {
+    slug: params.slug,
+    ...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {}
+  });
+}
+
+// src/ops/dataSourceEvals.ts
+var dataSourceEvals_exports = {};
+__export(dataSourceEvals_exports, {
+  DEFAULT_WAIT_TIMEOUT_MS: () => DEFAULT_WAIT_TIMEOUT_MS3,
+  addQueries: () => addQueries,
+  compare: () => compare,
+  createSet: () => createSet,
+  deleteQuery: () => deleteQuery,
+  deleteSet: () => deleteSet,
+  getRun: () => getRun,
+  getSet: () => getSet,
+  hydrate: () => hydrate,
+  listQueries: () => listQueries,
+  listRuns: () => listRuns,
+  listSets: () => listSets,
+  run: () => run2,
+  sample: () => sample,
+  waitForHydration: () => waitForHydration,
+  waitForRun: () => waitForRun,
+  waitForSet: () => waitForSet
+});
+var DEFAULT_WAIT_TIMEOUT_MS3 = 30 * 60 * 1e3;
+var POLL_MS3 = 4e3;
+function base4(appId) {
+  return `/_internal/v2/apps/${appId}/datasources`;
+}
+function hydrate(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base4(ctx.appId)}/hydrate`,
+    { slug: params.slug }
+  );
+}
+async function waitForHydration(ctx, params) {
+  const timeoutMs = params.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS3;
+  const outcome = await pollUntil(
+    async () => {
+      const { dataSources } = await list3(ctx);
+      const source2 = dataSources.find((s) => s.slug === params.slug);
+      if (!source2) {
+        throw new Error(`Data source "${params.slug}" not found.`);
+      }
+      return source2;
+    },
+    (source2) => !source2.hydration || source2.hydration.error !== null,
+    {
+      timeoutMs,
+      pollMs: POLL_MS3,
+      describe: (source2, start2) => `loading\u2026 ${source2.hydration.copied}/${source2.hydration.total} documents (${elapsedSeconds(start2)}s)`,
+      onProgress: params.onProgress
+    }
+  );
+  const source = outcome.value;
+  const hydration = source.hydration;
+  if (!outcome.settled) {
+    return {
+      status: "timeout",
+      source,
+      error: timedOut(
+        timeoutMs,
+        `with ${hydration.copied}/${hydration.total} documents loaded`
+      )
+    };
+  }
+  if (hydration?.error) {
+    return { status: "failed", source, error: hydration.error };
+  }
+  return { status: "done", source };
+}
+function sample(ctx, params) {
+  const { slug, size, as, filter, stratify, placement } = params;
+  return call(
+    ctx,
+    "POST",
+    `${base4(ctx.appId)}/sample`,
+    {
+      slug,
+      size,
+      ...as ? { as } : {},
+      ...filter ? { filter } : {},
+      ...stratify ? { stratify } : {},
+      ...placement !== void 0 ? { placement } : {}
+    }
+  );
+}
+function createSet(ctx, params) {
+  const { slug, name, description, size, style, modelId, tags, seed } = params;
+  return call(ctx, "POST", `${base4(ctx.appId)}/eval/sets`, {
+    slug,
+    name,
+    ...description ? { description } : {},
+    ...size !== void 0 ? { size } : {},
+    ...style ? { style } : {},
+    ...modelId ? { modelId } : {},
+    ...tags?.length ? { tags } : {},
+    ...seed !== void 0 ? { seed } : {}
+  });
+}
+function listSets(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base4(ctx.appId)}/eval/sets${qs({ slug: params?.slug })}`
+  );
+}
+function getSet(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base4(ctx.appId)}/eval/sets/${encodeURIComponent(params.id)}${qs({
+      queries: params.queries ? "true" : void 0
+    })}`
+  );
+}
+function deleteSet(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base4(ctx.appId)}/eval/sets/${encodeURIComponent(params.id)}/delete`
+  );
+}
+function addQueries(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base4(ctx.appId)}/eval/sets/${encodeURIComponent(params.id)}/queries`,
+    { queries: params.queries }
+  );
+}
+function listQueries(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base4(ctx.appId)}/eval/sets/${encodeURIComponent(params.id)}/queries`
+  );
+}
+function deleteQuery(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base4(ctx.appId)}/eval/sets/${encodeURIComponent(params.id)}/queries/delete`,
+    { queryId: params.queryId }
+  );
+}
+function run2(ctx, params) {
+  const { setId, slug, candidate, label, retrieval } = params;
+  return call(ctx, "POST", `${base4(ctx.appId)}/eval/runs`, {
+    setId,
+    ...slug ? { slug } : {},
+    ...candidate ? { candidate: true } : {},
+    ...label ? { label } : {},
+    ...retrieval && Object.keys(retrieval).length ? { retrieval } : {}
+  });
+}
+function listRuns(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base4(ctx.appId)}/eval/runs${qs({ setId: params?.setId, slug: params?.slug })}`
+  );
+}
+function getRun(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base4(ctx.appId)}/eval/runs/${encodeURIComponent(params.id)}${qs({
+      queries: params.queries ? "true" : void 0,
+      worst: params.worst
+    })}`
+  );
+}
+function compare(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base4(ctx.appId)}/eval/runs/compare${qs({ a: params.a, b: params.b })}`
+  );
+}
+async function waitForRun(ctx, params) {
+  const timeoutMs = params.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS3;
+  const outcome = await pollUntil(
+    async () => (await getRun(ctx, { id: params.id })).run,
+    (run4) => run4.state === "done" || run4.state === "failed",
+    {
+      timeoutMs,
+      pollMs: POLL_MS3,
+      describe: (run4, start2) => `${run4.state}\u2026 ${run4.completed}/${run4.total} queries (${elapsedSeconds(start2)}s)`,
+      onProgress: params.onProgress
+    }
+  );
+  const run3 = outcome.value;
+  if (!outcome.settled) {
+    return {
+      status: "timeout",
+      run: run3,
+      error: timedOut(
+        timeoutMs,
+        `; run is ${run3.state} (${run3.completed}/${run3.total})`
+      )
+    };
+  }
+  if (run3.state === "failed") {
+    return { status: "failed", run: run3, error: run3.error ?? "The run failed." };
+  }
+  return { status: "done", run: run3 };
+}
+async function waitForSet(ctx, params) {
+  const timeoutMs = params.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS3;
+  const outcome = await pollUntil(
+    async () => (await getSet(ctx, { id: params.id })).set,
+    (set4) => set4.state === "ready" || set4.state === "failed",
+    {
+      timeoutMs,
+      pollMs: POLL_MS3,
+      describe: (set4, start2) => `${set4.state}\u2026 (${elapsedSeconds(start2)}s)`,
+      onProgress: params.onProgress
+    }
+  );
+  const set3 = outcome.value;
+  if (!outcome.settled) {
+    return {
+      status: "timeout",
+      set: set3,
+      error: timedOut(timeoutMs, `; set is still ${set3.state}`)
+    };
+  }
+  if (set3.state === "failed") {
+    return { status: "failed", set: set3, error: set3.error ?? "Generation failed." };
+  }
+  return { status: "done", set: set3 };
+}
+
+// src/ops/dataSourceMappers.ts
+var dataSourceMappers_exports = {};
+__export(dataSourceMappers_exports, {
+  inspect: () => inspect,
+  mapTest: () => mapTest,
+  quarantine: () => quarantine,
+  remap: () => remap,
+  replay: () => replay
+});
+function base5(appId) {
+  return `/_internal/v2/apps/${appId}/datasources`;
+}
+function selectionBody(selection) {
+  if ("connector" in selection) {
+    return { connector: true };
+  }
+  return {
+    store: selection.store,
+    ...selection.access ? { access: selection.access } : {},
+    ...selection.prefix !== void 0 ? { prefix: selection.prefix } : {}
+  };
+}
+function inspect(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base5(ctx.appId)}/inspect`,
+    {
+      ...params.slug ? { slug: params.slug } : {},
+      ...selectionBody(params.selection),
+      ...params.keys ? { keys: params.keys } : {},
+      ...params.limit !== void 0 ? { limit: params.limit } : {}
+    }
+  );
+}
+function mapTest(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base5(ctx.appId)}/map-test`,
+    {
+      slug: params.slug,
+      ...selectionBody(params.selection),
+      ...params.keys ? { keys: params.keys } : {},
+      ...params.limit !== void 0 ? { limit: params.limit } : {},
+      ...params.dev ? { dev: true } : {}
+    }
+  );
+}
+function remap(ctx, params) {
+  return call(ctx, "POST", `${base5(ctx.appId)}/remap`, {
+    slug: params.slug,
+    ...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {},
+    ...params.limit !== void 0 ? { limit: params.limit } : {}
+  });
+}
+function quarantine(ctx, params) {
+  return call(
+    ctx,
+    "GET",
+    `${base5(ctx.appId)}/jobs/${params.id}/quarantine${qs({
+      ...params.kind ? { kind: params.kind } : {},
+      ...params.reason ? { reason: params.reason } : {},
+      ...params.cursor ? { cursor: params.cursor } : {},
+      ...params.limit !== void 0 ? { limit: params.limit } : {}
+    })}`
+  );
+}
+function replay(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base5(ctx.appId)}/jobs/${params.id}/replay`,
+    params.kind ? { kind: params.kind } : {}
+  );
+}
+
 // src/ops/db.ts
 var db_exports = {};
 __export(db_exports, {
@@ -749,18 +1343,18 @@ var releases_exports = {};
 __export(releases_exports, {
   byCommit: () => byCommit,
   dashboardLive: () => dashboardLive,
-  get: () => get2,
-  list: () => list4,
+  get: () => get4,
+  list: () => list5,
   waitForCommit: () => waitForCommit
 });
-function list4(ctx, params = {}) {
+function list5(ctx, params = {}) {
   return call(
     ctx,
     "GET",
     `/_internal/v2/apps/${ctx.appId}/releases${qs({ limit: params.limit })}`
   );
 }
-function get2(ctx, releaseId) {
+function get4(ctx, releaseId) {
   return call(
     ctx,
     "GET",
@@ -809,14 +1403,14 @@ function summarizeRelease(release) {
     ...release.failurePhase ? { failurePhase: release.failurePhase } : {}
   };
 }
-var POLL_MS2 = 3e3;
+var POLL_MS4 = 3e3;
 var RESOLVE_GRACE_MS = 3e4;
 var DEFAULT_TIMEOUT_MS = 3e5;
 async function waitForCommit(ctx, params) {
   const { commitSha, onProgress } = params;
   const timeoutMs = params.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   const shortSha = commitSha.slice(0, 8);
-  const start = Date.now();
+  const start2 = Date.now();
   let release;
   while (true) {
     const r = await byCommit(ctx, commitSha);
@@ -829,14 +1423,14 @@ async function waitForCommit(ctx, params) {
         `Failed to resolve release for ${shortSha}: HTTP ${r.status} ${JSON.stringify(r.body)}`
       );
     }
-    if (Date.now() - start > RESOLVE_GRACE_MS) {
+    if (Date.now() - start2 > RESOLVE_GRACE_MS) {
       return {
         outcome: "not_found",
         error: `No release created for commit ${commitSha} within ${RESOLVE_GRACE_MS / 1e3}s \u2014 either the push hasn't registered yet or this SHA never produced a build`
       };
     }
     onProgress?.(`waiting for release to be created for ${shortSha}\u2026`);
-    await sleep(POLL_MS2);
+    await sleep(POLL_MS4);
   }
   const SUCCESS = /* @__PURE__ */ new Set(["live", "preview"]);
   while (true) {
@@ -857,7 +1451,7 @@ async function waitForCommit(ctx, params) {
         summary: summarizeRelease(release)
       };
     }
-    if (Date.now() - start > timeoutMs) {
+    if (Date.now() - start2 > timeoutMs) {
       return {
         outcome: "timeout",
         release,
@@ -866,9 +1460,9 @@ async function waitForCommit(ctx, params) {
       };
     }
     onProgress?.(
-      `${release.status}\u2026 (${Math.round((Date.now() - start) / 1e3)}s)`
+      `${release.status}\u2026 (${Math.round((Date.now() - start2) / 1e3)}s)`
     );
-    await sleep(POLL_MS2);
+    await sleep(POLL_MS4);
     const r = await byCommit(ctx, commitSha);
     if (r.ok) {
       release = r.body;
@@ -911,10 +1505,10 @@ __export(domains_exports, {
   customRemove: () => customRemove,
   customRetry: () => customRetry,
   findHostname: () => findHostname,
-  get: () => get3,
+  get: () => get5,
   set: () => set
 });
-function get3(ctx) {
+function get5(ctx) {
   return call(
     ctx,
     "GET",
@@ -995,9 +1589,9 @@ __export(email_exports, {
   batch: () => batch2,
   batches: () => batches,
   checkDomain: () => checkDomain,
-  get: () => get4,
+  get: () => get6,
   inbox: () => inbox,
-  list: () => list5,
+  list: () => list6,
   listDomains: () => listDomains,
   removeDomain: () => removeDomain,
   stats: () => stats2,
@@ -1032,14 +1626,14 @@ async function resolveDomainId(ctx, direction, domain) {
   }
   return match.id;
 }
-function list5(ctx, params = {}) {
+function list6(ctx, params = {}) {
   return call(
     ctx,
     "GET",
     `/_internal/v2/apps/${ctx.appId}/outbound-email/messages${qs(params)}`
   );
 }
-function get4(ctx, messageId) {
+function get6(ctx, messageId) {
   return call(
     ctx,
     "GET",
@@ -1229,38 +1823,38 @@ function remove(ctx, ref) {
 // src/ops/infra.ts
 var infra_exports = {};
 __export(infra_exports, {
-  DEFAULT_WAIT_TIMEOUT_MS: () => DEFAULT_WAIT_TIMEOUT_MS2,
+  DEFAULT_WAIT_TIMEOUT_MS: () => DEFAULT_WAIT_TIMEOUT_MS4,
   destroy: () => destroy,
-  get: () => get5,
+  get: () => get7,
   hibernate: () => hibernate,
-  list: () => list6,
+  list: () => list7,
   provision: () => provision,
   rename: () => rename,
   resize: () => resize,
-  resume: () => resume,
+  resume: () => resume2,
   waitForPhase: () => waitForPhase
 });
-var DEFAULT_WAIT_TIMEOUT_MS2 = 10 * 60 * 1e3;
-var POLL_MS3 = 3e3;
-function base2(appId) {
+var DEFAULT_WAIT_TIMEOUT_MS4 = 10 * 60 * 1e3;
+var POLL_MS5 = 3e3;
+function base6(appId) {
   return `/_internal/v2/apps/${appId}/infra`;
 }
-function list6(ctx, params) {
+function list7(ctx, params) {
   return call(
     ctx,
     "GET",
-    `${base2(ctx.appId)}${params?.includeDestroyed ? "?includeDestroyed=true" : ""}`
+    `${base6(ctx.appId)}${params?.includeDestroyed ? "?includeDestroyed=true" : ""}`
   );
 }
-function get5(ctx, params) {
+function get7(ctx, params) {
   return call(
     ctx,
     "GET",
-    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}`
+    `${base6(ctx.appId)}/${encodeURIComponent(params.id)}`
   );
 }
 function provision(ctx, params) {
-  return call(ctx, "POST", base2(ctx.appId), {
+  return call(ctx, "POST", base6(ctx.appId), {
     offeringId: params.offeringId,
     ...params.name ? { name: params.name } : {}
   });
@@ -1269,28 +1863,28 @@ function hibernate(ctx, params) {
   return call(
     ctx,
     "POST",
-    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/hibernate`
+    `${base6(ctx.appId)}/${encodeURIComponent(params.id)}/hibernate`
   );
 }
-function resume(ctx, params) {
+function resume2(ctx, params) {
   return call(
     ctx,
     "POST",
-    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/resume`
+    `${base6(ctx.appId)}/${encodeURIComponent(params.id)}/resume`
   );
 }
 function destroy(ctx, params) {
   return call(
     ctx,
     "POST",
-    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/destroy`
+    `${base6(ctx.appId)}/${encodeURIComponent(params.id)}/destroy`
   );
 }
 function resize(ctx, params) {
   return call(
     ctx,
     "POST",
-    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/resize`,
+    `${base6(ctx.appId)}/${encodeURIComponent(params.id)}/resize`,
     { offeringId: params.offeringId }
   );
 }
@@ -1298,56 +1892,40 @@ function rename(ctx, params) {
   return call(
     ctx,
     "POST",
-    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/rename`,
+    `${base6(ctx.appId)}/${encodeURIComponent(params.id)}/rename`,
     { name: params.name }
   );
 }
 async function waitForPhase(ctx, params) {
-  const timeoutMs = params.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS2;
+  const timeoutMs = params.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS4;
   const wanted = new Set(params.phases);
-  const start = Date.now();
-  let transientSince = null;
-  for (; ; ) {
-    let resource;
-    try {
-      ({ resource } = await get5(ctx, { id: params.id }));
-      transientSince = null;
-    } catch (err) {
-      if (!isTransientError(err)) {
-        throw err;
-      }
-      transientSince ??= Date.now();
-      if (Date.now() - transientSince > TRANSIENT_GRACE_MS) {
-        throw err;
-      }
-      params.onProgress?.(
-        `retrying after a gateway error\u2026 (${Math.round((Date.now() - start) / 1e3)}s)`
-      );
-      await sleep(POLL_MS3);
-      continue;
-    }
-    if (wanted.has(resource.phase) && (params.until?.(resource) ?? true)) {
-      return { status: "done", resource };
-    }
-    if (resource.phase === "failed") {
-      return {
-        status: "failed",
-        resource,
-        error: resource.lastError ?? "The platform could not complete the change."
-      };
-    }
-    if (Date.now() - start > timeoutMs) {
-      return {
-        status: "timeout",
-        resource,
-        error: `Timed out after ${Math.round(timeoutMs / 1e3)}s; resource is still ${resource.phase}`
-      };
+  const arrived = (resource2) => wanted.has(resource2.phase) && (params.until?.(resource2) ?? true);
+  const outcome = await pollUntil(
+    async () => (await get7(ctx, { id: params.id })).resource,
+    (resource2) => arrived(resource2) || resource2.phase === "failed",
+    {
+      timeoutMs,
+      pollMs: POLL_MS5,
+      describe: (resource2, start2) => `${resource2.phase}${resource2.detail ? ` \xB7 ${resource2.detail}` : ""}${resource2.resizingTo ? ` \xB7 resizing to ${resource2.resizingTo.label}` : ""}\u2026 (${elapsedSeconds(start2)}s)`,
+      onProgress: params.onProgress
     }
-    params.onProgress?.(
-      `${resource.phase}${resource.detail ? ` \xB7 ${resource.detail}` : ""}${resource.resizingTo ? ` \xB7 resizing to ${resource.resizingTo.label}` : ""}\u2026 (${Math.round((Date.now() - start) / 1e3)}s)`
-    );
-    await sleep(POLL_MS3);
+  );
+  const resource = outcome.value;
+  if (!outcome.settled) {
+    return {
+      status: "timeout",
+      resource,
+      error: timedOut(timeoutMs, `; resource is still ${resource.phase}`)
+    };
+  }
+  if (arrived(resource)) {
+    return { status: "done", resource };
   }
+  return {
+    status: "failed",
+    resource,
+    error: resource.lastError ?? "The platform could not complete the change."
+  };
 }
 
 // src/ops/issues.ts
@@ -1358,18 +1936,18 @@ __export(issues_exports, {
   create: () => create2,
   del: () => del,
   edit: () => edit,
-  get: () => get6,
-  list: () => list7,
+  get: () => get8,
+  list: () => list8,
   reopen: () => reopen
 });
-function list7(ctx, params = {}) {
+function list8(ctx, params = {}) {
   return call(
     ctx,
     "GET",
     `/_internal/v2/apps/${ctx.appId}/issues${qs(params)}`
   );
 }
-function get6(ctx, number) {
+function get8(ctx, number) {
   return call(
     ctx,
     "GET",
@@ -1453,7 +2031,7 @@ __export(jewels_exports, {
   JEWEL_RUN_TIMEOUT_MS: () => JEWEL_RUN_TIMEOUT_MS,
   dryrun: () => dryrun,
   exportSummary: () => exportSummary,
-  getRun: () => getRun,
+  getRun: () => getRun2,
   grade: () => grade,
   overview: () => overview,
   pair: () => pair,
@@ -1550,7 +2128,7 @@ function runs(ctx, params = {}) {
     `/_internal/v2/apps/${ctx.appId}/jewels/training-runs${qs(params)}`
   );
 }
-function getRun(ctx, runId) {
+function getRun2(ctx, runId) {
   return call(
     ctx,
     "GET",
@@ -1571,9 +2149,9 @@ function grade(ctx, runId) {
 var methods_exports = {};
 __export(methods_exports, {
   invoke: () => invoke,
-  list: () => list8
+  list: () => list9
 });
-async function list8(ctx) {
+async function list9(ctx) {
   const dashboard = await call(
     ctx,
     "GET",
@@ -1630,19 +2208,19 @@ function invalidate(ctx, params = {}) {
 // src/ops/requests.ts
 var requests_exports = {};
 __export(requests_exports, {
-  get: () => get7,
-  list: () => list9,
+  get: () => get9,
+  list: () => list10,
   statsForMethod: () => statsForMethod,
   statsSummary: () => statsSummary
 });
-function list9(ctx, params = {}) {
+function list10(ctx, params = {}) {
   return call(
     ctx,
     "GET",
     `/_internal/v2/apps/${ctx.appId}/requests${qs(params)}`
   );
 }
-function get7(ctx, requestId) {
+function get9(ctx, requestId) {
   return call(
     ctx,
     "GET",
@@ -1668,18 +2246,18 @@ function statsForMethod(ctx, methodId, params = {}) {
 var secrets_exports = {};
 __export(secrets_exports, {
   del: () => del2,
-  get: () => get8,
-  list: () => list10,
+  get: () => get10,
+  list: () => list11,
   set: () => set2
 });
-function list10(ctx) {
+function list11(ctx) {
   return call(
     ctx,
     "GET",
     `/_internal/v2/apps/${ctx.appId}/secrets`
   );
 }
-function get8(ctx, key) {
+function get10(ctx, key) {
   return call(
     ctx,
     "GET",
@@ -1735,11 +2313,11 @@ function updateSettings(ctx, partial) {
 var users_exports = {};
 __export(users_exports, {
   createApiKey: () => createApiKey,
-  list: () => list11,
+  list: () => list12,
   revokeApiKey: () => revokeApiKey,
   setRole: () => setRole
 });
-function list11(ctx, params = {}) {
+function list12(ctx, params = {}) {
   return call(
     ctx,
     "GET",
@@ -1925,6 +2503,10 @@ function buildClient(ctx) {
     cron: bindOps(cron_exports, ctx),
     data: bindOps(data_exports, ctx),
     dataSources: bindOps(dataSources_exports, ctx),
+    dataSourceJobs: bindOps(dataSourceJobs_exports, ctx),
+    dataSourceConnectors: bindOps(dataSourceConnectors_exports, ctx),
+    dataSourceEvals: bindOps(dataSourceEvals_exports, ctx),
+    dataSourceMappers: bindOps(dataSourceMappers_exports, ctx),
     db: bindOps(db_exports, ctx),
     diagnostics: bindOps(diagnostics_exports, ctx),
     domains: bindOps(domains_exports, ctx),