@madewithremy/admin 0.1.67 → 0.1.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -4,6 +4,17 @@ What changed in `@madewithremy/admin`, newest first, each version under its own
4
4
 
5
5
  Versions before 0.1.44 aren't recorded here.
6
6
 
7
+ ## 0.1.68
8
+
9
+ ### New
10
+
11
+ - `datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>` adds a file of documents, one per line as `{ "externalId", "title", "markdown", "metadata"?, "entities"?, "replaces"? }`, built as one job that plans and then starts on its own within the budget. From code it's `admin.dataSourceJobs.addMany({ slug, documents })`. It's for records rendered to text, in any number; a file is still `datasources add`. Check: a script that writes many generated text files and adds each with `datasources add` can write one JSONL file and use this.
12
+
13
+ ### Changed behavior
14
+
15
+ - `datasources list` doesn't return `documentLimit` on a source, and `datasources add` isn't refused with `document_limit_reached`: a source holds as many documents as the place it lives has room for. Past that, `add` answers `data_source_full` (422), and the message says whether to move the source to dedicated capacity or resize it. Check: a script that reads `documentLimit` or matches `document_limit_reached`.
16
+ - A job's `source` may be `{ type: 'documents', documents }`, the documents an `add-many` gave. Check: code that switches over a job's `source.type`.
17
+
7
18
  ## 0.1.67
8
19
 
9
20
  ### Changed behavior
package/dist/index.js CHANGED
@@ -1143,6 +1143,7 @@ var drop = dropPipelineVersion;
1143
1143
  var dataSourceJobs_exports = {};
1144
1144
  __export(dataSourceJobs_exports, {
1145
1145
  DEFAULT_WAIT_TIMEOUT_MS: () => DEFAULT_WAIT_TIMEOUT_MS2,
1146
+ addMany: () => addMany,
1146
1147
  approve: () => approve,
1147
1148
  cancel: () => cancel,
1148
1149
  describeProgress: () => describeProgress,
@@ -1156,6 +1157,7 @@ __export(dataSourceJobs_exports, {
1156
1157
  waitForJob: () => waitForJob,
1157
1158
  waitForPlan: () => waitForPlan
1158
1159
  });
1160
+ import { gzipSync } from "node:zlib";
1159
1161
  var DEFAULT_WAIT_TIMEOUT_MS2 = 60 * 60 * 1e3;
1160
1162
  var POLL_MS2 = 5e3;
1161
1163
  function base2(appId) {
@@ -1183,6 +1185,66 @@ async function uploadManifest(ctx, params) {
1183
1185
  await uploadDirect(token.upload, params.content, "manifest.jsonl");
1184
1186
  return { key: token.key };
1185
1187
  }
1188
+ var PART_DOCUMENTS = 1e3;
1189
+ var PART_BYTES = 8 * 1024 * 1024;
1190
+ async function addMany(ctx, params) {
1191
+ const addManyBase = `/_internal/v2/apps/${ctx.appId}/datasources/add-many`;
1192
+ const { uploadId } = await call(
1193
+ ctx,
1194
+ "POST",
1195
+ `${addManyBase}/open`,
1196
+ { slug: params.slug }
1197
+ );
1198
+ const parts = [];
1199
+ let lines = [];
1200
+ let bytes = 0;
1201
+ let documentCount = 0;
1202
+ const storeLines = async () => {
1203
+ if (lines.length === 0) {
1204
+ return;
1205
+ }
1206
+ const grant = await call(
1207
+ ctx,
1208
+ "POST",
1209
+ `${addManyBase}/part`,
1210
+ { slug: params.slug, uploadId }
1211
+ );
1212
+ await uploadDirect(
1213
+ grant,
1214
+ gzipSync(Buffer.from(lines.join("\n") + "\n")),
1215
+ `${grant.part}.ndjson.gz`
1216
+ );
1217
+ parts.push({ part: grant.part, count: lines.length });
1218
+ lines = [];
1219
+ bytes = 0;
1220
+ params.onProgress?.(documentCount);
1221
+ };
1222
+ for await (const d of params.documents) {
1223
+ const line = JSON.stringify({ o: 0, i: documentCount, d });
1224
+ lines.push(line);
1225
+ bytes += line.length;
1226
+ documentCount += 1;
1227
+ if (lines.length >= PART_DOCUMENTS || bytes >= PART_BYTES) {
1228
+ await storeLines();
1229
+ }
1230
+ }
1231
+ await storeLines();
1232
+ if (parts.length === 0) {
1233
+ return { documentCount: 0, job: null };
1234
+ }
1235
+ const { job } = await call(
1236
+ ctx,
1237
+ "POST",
1238
+ `${addManyBase}/start`,
1239
+ {
1240
+ slug: params.slug,
1241
+ uploadId,
1242
+ parts,
1243
+ ...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {}
1244
+ }
1245
+ );
1246
+ return { documentCount, job };
1247
+ }
1186
1248
  function list6(ctx, params) {
1187
1249
  return call(
1188
1250
  ctx,