@madewithremy/admin 0.1.67 → 0.1.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/index.js +62 -0
- package/dist/index.js.map +2 -2
- package/dist/ops/dataSourceJobs.d.ts +52 -1
- package/dist/ops/dataSources.d.ts +4 -4
- package/dist/ops/db.d.ts +8 -3
- package/dist/prod.js +167 -3
- package/dist/types/dataSourceJobs.d.ts +23 -0
- package/dist/types/dataSources.d.ts +0 -5
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,23 @@ What changed in `@madewithremy/admin`, newest first, each version under its own
|
|
|
4
4
|
|
|
5
5
|
Versions before 0.1.44 aren't recorded here.
|
|
6
6
|
|
|
7
|
+
## 0.1.69
|
|
8
|
+
|
|
9
|
+
### Changed behavior
|
|
10
|
+
|
|
11
|
+
- A `db query` or `db history attach` that changes the live database is recorded in the workspace's audit log, as file uploads, file deletes and `db history purge` already are: who ran it and when, which database, how many statements and how many rows changed, never the SQL. A read isn't recorded, and neither is anything run against a dev database. `db --help` says so. Check: nothing to change in a script; a bulk write sent one statement at a time shows as one line per statement in Workspace Settings' Audit Log.
|
|
12
|
+
|
|
13
|
+
## 0.1.68
|
|
14
|
+
|
|
15
|
+
### New
|
|
16
|
+
|
|
17
|
+
- `datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>` adds a file of documents, one per line as `{ "externalId", "title", "markdown", "metadata"?, "entities"?, "replaces"? }`, built as one job that plans and then starts on its own within the budget. From code it's `admin.dataSourceJobs.addMany({ slug, documents })`. It's for records rendered to text, in any number; a file is still `datasources add`. Check: a script that writes many generated text files and adds each with `datasources add` can write one JSONL file and use this.
|
|
18
|
+
|
|
19
|
+
### Changed behavior
|
|
20
|
+
|
|
21
|
+
- `datasources list` doesn't return `documentLimit` on a source, and `datasources add` isn't refused with `document_limit_reached`: a source holds as many documents as the place it lives has room for. Past that, `add` answers `data_source_full` (422), and the message says whether to move the source to dedicated capacity or resize it. Check: a script that reads `documentLimit` or matches `document_limit_reached`.
|
|
22
|
+
- A job's `source` may be `{ type: 'documents', documents }`, the documents an `add-many` gave. Check: code that switches over a job's `source.type`.
|
|
23
|
+
|
|
7
24
|
## 0.1.67
|
|
8
25
|
|
|
9
26
|
### Changed behavior
|
package/dist/index.js
CHANGED
|
@@ -1143,6 +1143,7 @@ var drop = dropPipelineVersion;
|
|
|
1143
1143
|
var dataSourceJobs_exports = {};
|
|
1144
1144
|
__export(dataSourceJobs_exports, {
|
|
1145
1145
|
DEFAULT_WAIT_TIMEOUT_MS: () => DEFAULT_WAIT_TIMEOUT_MS2,
|
|
1146
|
+
addMany: () => addMany,
|
|
1146
1147
|
approve: () => approve,
|
|
1147
1148
|
cancel: () => cancel,
|
|
1148
1149
|
describeProgress: () => describeProgress,
|
|
@@ -1156,6 +1157,7 @@ __export(dataSourceJobs_exports, {
|
|
|
1156
1157
|
waitForJob: () => waitForJob,
|
|
1157
1158
|
waitForPlan: () => waitForPlan
|
|
1158
1159
|
});
|
|
1160
|
+
import { gzipSync } from "node:zlib";
|
|
1159
1161
|
var DEFAULT_WAIT_TIMEOUT_MS2 = 60 * 60 * 1e3;
|
|
1160
1162
|
var POLL_MS2 = 5e3;
|
|
1161
1163
|
function base2(appId) {
|
|
@@ -1183,6 +1185,66 @@ async function uploadManifest(ctx, params) {
|
|
|
1183
1185
|
await uploadDirect(token.upload, params.content, "manifest.jsonl");
|
|
1184
1186
|
return { key: token.key };
|
|
1185
1187
|
}
|
|
1188
|
+
var PART_DOCUMENTS = 1e3;
|
|
1189
|
+
var PART_BYTES = 8 * 1024 * 1024;
|
|
1190
|
+
async function addMany(ctx, params) {
|
|
1191
|
+
const addManyBase = `/_internal/v2/apps/${ctx.appId}/datasources/add-many`;
|
|
1192
|
+
const { uploadId } = await call(
|
|
1193
|
+
ctx,
|
|
1194
|
+
"POST",
|
|
1195
|
+
`${addManyBase}/open`,
|
|
1196
|
+
{ slug: params.slug }
|
|
1197
|
+
);
|
|
1198
|
+
const parts = [];
|
|
1199
|
+
let lines = [];
|
|
1200
|
+
let bytes = 0;
|
|
1201
|
+
let documentCount = 0;
|
|
1202
|
+
const storeLines = async () => {
|
|
1203
|
+
if (lines.length === 0) {
|
|
1204
|
+
return;
|
|
1205
|
+
}
|
|
1206
|
+
const grant = await call(
|
|
1207
|
+
ctx,
|
|
1208
|
+
"POST",
|
|
1209
|
+
`${addManyBase}/part`,
|
|
1210
|
+
{ slug: params.slug, uploadId }
|
|
1211
|
+
);
|
|
1212
|
+
await uploadDirect(
|
|
1213
|
+
grant,
|
|
1214
|
+
gzipSync(Buffer.from(lines.join("\n") + "\n")),
|
|
1215
|
+
`${grant.part}.ndjson.gz`
|
|
1216
|
+
);
|
|
1217
|
+
parts.push({ part: grant.part, count: lines.length });
|
|
1218
|
+
lines = [];
|
|
1219
|
+
bytes = 0;
|
|
1220
|
+
params.onProgress?.(documentCount);
|
|
1221
|
+
};
|
|
1222
|
+
for await (const d of params.documents) {
|
|
1223
|
+
const line = JSON.stringify({ o: 0, i: documentCount, d });
|
|
1224
|
+
lines.push(line);
|
|
1225
|
+
bytes += line.length;
|
|
1226
|
+
documentCount += 1;
|
|
1227
|
+
if (lines.length >= PART_DOCUMENTS || bytes >= PART_BYTES) {
|
|
1228
|
+
await storeLines();
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
await storeLines();
|
|
1232
|
+
if (parts.length === 0) {
|
|
1233
|
+
return { documentCount: 0, job: null };
|
|
1234
|
+
}
|
|
1235
|
+
const { job } = await call(
|
|
1236
|
+
ctx,
|
|
1237
|
+
"POST",
|
|
1238
|
+
`${addManyBase}/start`,
|
|
1239
|
+
{
|
|
1240
|
+
slug: params.slug,
|
|
1241
|
+
uploadId,
|
|
1242
|
+
parts,
|
|
1243
|
+
...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {}
|
|
1244
|
+
}
|
|
1245
|
+
);
|
|
1246
|
+
return { documentCount, job };
|
|
1247
|
+
}
|
|
1186
1248
|
function list6(ctx, params) {
|
|
1187
1249
|
return call(
|
|
1188
1250
|
ctx,
|