@madewithremy/admin 0.1.66 → 0.1.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/index.js +62 -0
- package/dist/index.js.map +2 -2
- package/dist/ops/dataSourceJobs.d.ts +52 -1
- package/dist/ops/dataSources.d.ts +4 -4
- package/dist/prod.js +166 -10
- package/dist/types/dataSourceJobs.d.ts +23 -0
- package/dist/types/dataSources.d.ts +0 -5
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,23 @@ What changed in `@madewithremy/admin`, newest first, each version under its own
|
|
|
4
4
|
|
|
5
5
|
Versions before 0.1.44 aren't recorded here.
|
|
6
6
|
|
|
7
|
+
## 0.1.68
|
|
8
|
+
|
|
9
|
+
### New
|
|
10
|
+
|
|
11
|
+
- `datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>` adds a file of documents, one per line as `{ "externalId", "title", "markdown", "metadata"?, "entities"?, "replaces"? }`, built as one job that plans and then starts on its own within the budget. From code it's `admin.dataSourceJobs.addMany({ slug, documents })`. It's for records rendered to text, in any number; a file is still `datasources add`. Check: a script that writes many generated text files and adds each with `datasources add` can write one JSONL file and use this.
|
|
12
|
+
|
|
13
|
+
### Changed behavior
|
|
14
|
+
|
|
15
|
+
- `datasources list` doesn't return `documentLimit` on a source, and `datasources add` isn't refused with `document_limit_reached`: a source holds as many documents as the place it lives has room for. Past that, `add` answers `data_source_full` (422), and the message says whether to move the source to dedicated capacity or resize it. Check: a script that reads `documentLimit` or matches `document_limit_reached`.
|
|
16
|
+
- A job's `source` may be `{ type: 'documents', documents }`, the documents an `add-many` gave. Check: code that switches over a job's `source.type`.
|
|
17
|
+
|
|
18
|
+
## 0.1.67
|
|
19
|
+
|
|
20
|
+
### Changed behavior
|
|
21
|
+
|
|
22
|
+
- `jewels train --wait` waits until the run ends, however long that takes: the platform ends every run, by the trainer's report or by failing one that has gone quiet. It used to give up after 45 minutes with "Timed out waiting for run", though a run may take two hours. Check: a script that relied on the 45-minute exit should poll `jewels run <runId>` itself and stop when it chooses.
|
|
23
|
+
|
|
7
24
|
## 0.1.65
|
|
8
25
|
|
|
9
26
|
### New
|
package/dist/index.js
CHANGED
|
@@ -1143,6 +1143,7 @@ var drop = dropPipelineVersion;
|
|
|
1143
1143
|
var dataSourceJobs_exports = {};
|
|
1144
1144
|
__export(dataSourceJobs_exports, {
|
|
1145
1145
|
DEFAULT_WAIT_TIMEOUT_MS: () => DEFAULT_WAIT_TIMEOUT_MS2,
|
|
1146
|
+
addMany: () => addMany,
|
|
1146
1147
|
approve: () => approve,
|
|
1147
1148
|
cancel: () => cancel,
|
|
1148
1149
|
describeProgress: () => describeProgress,
|
|
@@ -1156,6 +1157,7 @@ __export(dataSourceJobs_exports, {
|
|
|
1156
1157
|
waitForJob: () => waitForJob,
|
|
1157
1158
|
waitForPlan: () => waitForPlan
|
|
1158
1159
|
});
|
|
1160
|
+
import { gzipSync } from "node:zlib";
|
|
1159
1161
|
var DEFAULT_WAIT_TIMEOUT_MS2 = 60 * 60 * 1e3;
|
|
1160
1162
|
var POLL_MS2 = 5e3;
|
|
1161
1163
|
function base2(appId) {
|
|
@@ -1183,6 +1185,66 @@ async function uploadManifest(ctx, params) {
|
|
|
1183
1185
|
await uploadDirect(token.upload, params.content, "manifest.jsonl");
|
|
1184
1186
|
return { key: token.key };
|
|
1185
1187
|
}
|
|
1188
|
+
var PART_DOCUMENTS = 1e3;
|
|
1189
|
+
var PART_BYTES = 8 * 1024 * 1024;
|
|
1190
|
+
async function addMany(ctx, params) {
|
|
1191
|
+
const addManyBase = `/_internal/v2/apps/${ctx.appId}/datasources/add-many`;
|
|
1192
|
+
const { uploadId } = await call(
|
|
1193
|
+
ctx,
|
|
1194
|
+
"POST",
|
|
1195
|
+
`${addManyBase}/open`,
|
|
1196
|
+
{ slug: params.slug }
|
|
1197
|
+
);
|
|
1198
|
+
const parts = [];
|
|
1199
|
+
let lines = [];
|
|
1200
|
+
let bytes = 0;
|
|
1201
|
+
let documentCount = 0;
|
|
1202
|
+
const storeLines = async () => {
|
|
1203
|
+
if (lines.length === 0) {
|
|
1204
|
+
return;
|
|
1205
|
+
}
|
|
1206
|
+
const grant = await call(
|
|
1207
|
+
ctx,
|
|
1208
|
+
"POST",
|
|
1209
|
+
`${addManyBase}/part`,
|
|
1210
|
+
{ slug: params.slug, uploadId }
|
|
1211
|
+
);
|
|
1212
|
+
await uploadDirect(
|
|
1213
|
+
grant,
|
|
1214
|
+
gzipSync(Buffer.from(lines.join("\n") + "\n")),
|
|
1215
|
+
`${grant.part}.ndjson.gz`
|
|
1216
|
+
);
|
|
1217
|
+
parts.push({ part: grant.part, count: lines.length });
|
|
1218
|
+
lines = [];
|
|
1219
|
+
bytes = 0;
|
|
1220
|
+
params.onProgress?.(documentCount);
|
|
1221
|
+
};
|
|
1222
|
+
for await (const d of params.documents) {
|
|
1223
|
+
const line = JSON.stringify({ o: 0, i: documentCount, d });
|
|
1224
|
+
lines.push(line);
|
|
1225
|
+
bytes += line.length;
|
|
1226
|
+
documentCount += 1;
|
|
1227
|
+
if (lines.length >= PART_DOCUMENTS || bytes >= PART_BYTES) {
|
|
1228
|
+
await storeLines();
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
await storeLines();
|
|
1232
|
+
if (parts.length === 0) {
|
|
1233
|
+
return { documentCount: 0, job: null };
|
|
1234
|
+
}
|
|
1235
|
+
const { job } = await call(
|
|
1236
|
+
ctx,
|
|
1237
|
+
"POST",
|
|
1238
|
+
`${addManyBase}/start`,
|
|
1239
|
+
{
|
|
1240
|
+
slug: params.slug,
|
|
1241
|
+
uploadId,
|
|
1242
|
+
parts,
|
|
1243
|
+
...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {}
|
|
1244
|
+
}
|
|
1245
|
+
);
|
|
1246
|
+
return { documentCount, job };
|
|
1247
|
+
}
|
|
1186
1248
|
function list6(ctx, params) {
|
|
1187
1249
|
return call(
|
|
1188
1250
|
ctx,
|