@madewithremy/admin 0.1.32 → 0.1.34

package/dist/prod.js

@@ -1019,13 +1019,6 @@ async function dashboardLive(ctx) {
   );
   return dashboard.liveRelease;
 }
-function buildFailureReason(release) {
-  const log = Array.isArray(release.buildLog) ? release.buildLog : [];
-  const errors = log.filter(
-    (entry) => entry?.phase === "error" && typeof entry.message === "string"
-  );
-  return errors.length ? errors[errors.length - 1].message : null;
-}
 function summarizeRelease(release) {
   const summary2 = {
     releaseId: release.id,
@@ -1048,7 +1041,7 @@ function summarizeRelease(release) {
   }
   return {
     ...summary2,
-    error: release.failureReason ?? buildFailureReason(release) ?? "Build failed (no error entry in the build log)",
+    error: release.failureReason ?? "Build failed (no error entry in the build log)",
     ...release.failureKind ? { failureKind: release.failureKind } : {},
     ...release.failurePhase ? { failurePhase: release.failurePhase } : {}
   };
@@ -3171,6 +3164,7 @@ var summarize = (d) => ({
 });
 async function addDocument(ctx, params) {
   const { slug, filename, content, metadata, entities: entities2, onProgress } = params;
+  const { budgetDollars } = params;
   const contentHash = createHash2("sha256").update(content).digest("hex");
   const token = await call(
     ctx,
@@ -3186,11 +3180,13 @@ async function addDocument(ctx, params) {
   );
   if (token.alreadyCurrent) {
     onProgress?.(`${filename}: unchanged, skipped`);
+    const documents3 = token.documents ?? [token.document];
     return {
       filename,
       skipped: true,
       document: token.document,
-      documents: token.documents ?? [token.document]
+      documents: documents3,
+      documentCount: token.documentCount ?? documents3.length
     };
   }
   onProgress?.(
@@ -3208,16 +3204,21 @@ async function addDocument(ctx, params) {
       contentType: token.contentType,
       size: content.length,
       ...metadata ? { metadata } : {},
-      ...entities2 ? { entities: entities2 } : {}
+      ...entities2 ? { entities: entities2 } : {},
+      ...budgetDollars !== void 0 ? { budgetDollars } : {}
     }
   );
+  const documents2 = confirmed.documents ?? (confirmed.document ? [confirmed.document] : []);
   return {
     filename,
     skipped: false,
     queued: confirmed.queued,
     document: confirmed.document,
-    documents: confirmed.documents ?? (confirmed.document ? [confirmed.document] : []),
-    outcome: confirmed.outcome
+    documents: documents2,
+    documentCount: confirmed.documentCount ?? documents2.length,
+    outcome: confirmed.outcome,
+    invalid: confirmed.invalid ?? null,
+    job: confirmed.job ?? null
   };
 }
 async function waitForIngest(ctx, params) {
@@ -5583,14 +5584,21 @@ Mapping raw objects into documents (mappers):
   map(object) reads the raw object (object.key, .bytes(), .text(), .json())
   and answers documents([{ externalId, title, markdown, metadata?, replaces? }]),
   passthrough() (ingest the raw object as-is), skip(reason) or
-  deletes([externalId, ...]); throwing is the object's error. It may call
-  models or fetch \u2014 every call is spend per object. Everything entering a
-  mapped source is mapped: jobs, syncs and Source.add() alike.
+  deletes([externalId, ...]); throwing is the object's error. documents()
+  takes an array or any iterable, with no cap: the platform stores what it
+  produces as it goes, so a generator yielding a document per row streams an
+  export of any size. It may call models or fetch \u2014 every call is spend per
+  object. Everything entering
+  a mapped source is mapped: jobs, syncs and Source.add() alike; an add() of
+  an object mapping to more than a thousand documents registers the first
+  thousand and hands the rest to a job (\`datasources add --budget\` caps it).
 
   The loop: \`inspect\` profiles the objects (key shapes, extensions, sizes,
   JSON key signatures of ten sampled heads) before a line is written;
   \`map test --dev\` runs the LOCAL mapper through the running dev session
-  over real objects and prints every outcome without ingesting anything;
+  over real objects and prints every outcome without ingesting anything
+  (each object's count, its first documents, and every document that fails
+  validation, by its place in the mapper's output);
   push a branch and wait for its build (\`releases wait\`) \u2014 a mapper runs on
   the platform, so the platform has to compile it, and a private branch
   build is all it needs; \`map deploy\` makes that build's mapper the
@@ -5609,7 +5617,11 @@ Mapping raw objects into documents (mappers):
   ids this document supersedes (a writethru's original); deletes([...])
   removes by id. A plan on a mapped source records the mapper's outcome mix
   on its sample; a run whose skip share climbs past twice that (floor 5%)
-  pauses with pauseReason 'skips' for a look at the quarantine.
+  pauses with pauseReason 'skips' for a look at the quarantine. Documents
+  that fail validation are left out and their object gets one quarantine row
+  naming them; the rest register. A mapper built with an older
+  @mindstudio-ai/agent, which returns documents inline, errors every
+  documents outcome until it is upgraded, pushed and map deployed.
 `;
 
 // src/ops/dataSourceReaders.ts
@@ -5870,12 +5882,13 @@ var CONFIG_FLAGS = {
 };
 var dataSourcesSpecs = {
   "datasources add": {
-    usage: "Usage: remy-admin datasources add [--source <slug>] [--metadata <k=v,...>] [--entities <id,...>] [--wait] [--timeout <sec>] <file...>",
+    usage: "Usage: remy-admin datasources add [--source <slug>] [--metadata <k=v,...>] [--entities <id,...>] [--budget <dollars>] [--wait] [--timeout <sec>] <file...>",
     positionals: [{ name: "file", required: true, variadic: true }],
     flags: {
       source: { type: "string" },
       metadata: { type: "string" },
       entities: { type: "string" },
+      budget: { type: "number", min: 0.01 },
       wait: { type: "boolean" },
       timeout: { type: "string" }
     }
@@ -6019,6 +6032,7 @@ async function dataSourcesAdd(ctx, a) {
   const slug = sourceOf(a);
   const metadata = parseKeyValuePairs(a.str("metadata"), "metadata");
   const entities2 = parseIdList(a.str("entities"));
+  const budgetDollars = a.num("budget");
   const results = [];
   for (const file of files) {
     const abs = path6.isAbsolute(file) ? file : path6.join(WORKSPACE_DIR, file);
@@ -6036,10 +6050,22 @@ async function dataSourcesAdd(ctx, a) {
         content: bytes,
         ...metadata ? { metadata } : {},
         ...entities2 ? { entities: entities2 } : {},
+        ...budgetDollars !== void 0 ? { budgetDollars } : {},
         onProgress: progress
       })
     );
   }
+  const jobs = results.flatMap(
+    (r) => r.job ? [
+      {
+        file: r.filename,
+        documents: r.documentCount,
+        job: r.job.id,
+        state: r.job.state,
+        follow: `remy-admin datasources jobs status ${r.job.id}`
+      }
+    ] : []
+  );
   if (a.bool("wait")) {
     const timeoutSec = a.num("timeout");
     const timeoutMs = timeoutSec ? timeoutSec * 1e3 : DEFAULT_WAIT_TIMEOUT_MS;
@@ -6055,7 +6081,8 @@ async function dataSourcesAdd(ctx, a) {
       dataSource: slug,
       status: result.status,
       ...result.error ? { error: result.error } : {},
-      documents: result.documents
+      documents: result.documents,
+      ...jobs.length > 0 ? { jobs } : {}
     });
     if (result.status === "timeout") {
       throw new CliError(result.error ?? "Timed out.", EXIT.timeout);
@@ -6065,7 +6092,11 @@ async function dataSourcesAdd(ctx, a) {
     }
     return;
   }
-  out({ dataSource: slug, documents: results });
+  out({
+    dataSource: slug,
+    documents: results.map(({ job, ...r }) => r),
+    ...jobs.length > 0 ? { jobs } : {}
+  });
 }
 async function dataSourcesList(ctx) {
   out(await list8(ctx));
@@ -6516,7 +6547,7 @@ Subcommands:
   delete       Delete a whole data source \u2014 documents, vectors and versions
 
 Usage:
-  remy-admin datasources add [--source <slug>] [--metadata <k=v,...>] [--entities <id,...>] [--wait] [--timeout <sec>] <file...>
+  remy-admin datasources add [--source <slug>] [--metadata <k=v,...>] [--entities <id,...>] [--budget <dollars>] [--wait] [--timeout <sec>] <file...>
   remy-admin datasources list
   remy-admin datasources status [--source <slug>] [--status error] [--all]
   remy-admin datasources rm [--source <slug>] (--document <id> | --filter <k=v,...|json>)
@@ -6657,6 +6688,10 @@ Notes:
   --metadata tags documents for search-time filtering (scalars, \u226416 keys).
   --entities names what a document is about, as your own ids (\u2264256, no spaces
     or commas); an empty --entities clears them. Also free on a re-add.
+  On a mapped source, a file that maps to more than a thousand documents adds
+    the first thousand and starts a job for the rest, listed under \`jobs\` in
+    the output; --budget caps that job. --wait waits for the first thousand;
+    follow the job with \`jobs status <id> --wait\`.
   rm --filter removes every matching document in pages of a thousand: k=v
     pairs match metadata; a JSON object takes the full selector \u2014 metadata,
     filename, documentIds, externalIdPrefix (the key a job or connector

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@madewithremy/admin",
-  "version": "0.1.32",
+  "version": "0.1.34",
   "description": "Admin SDK + CLI for managing production Remy apps",
   "type": "module",
   "main": "./dist/index.js",