@madewithremy/admin 0.1.66 → 0.1.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/index.js +62 -0
- package/dist/index.js.map +2 -2
- package/dist/ops/dataSourceJobs.d.ts +52 -1
- package/dist/ops/dataSources.d.ts +4 -4
- package/dist/prod.js +166 -10
- package/dist/types/dataSourceJobs.d.ts +23 -0
- package/dist/types/dataSources.d.ts +0 -5
- package/package.json +1 -1
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* commands/dataSourceJobs.ts owns flags, progress printing and exit codes.
|
|
8
8
|
*/
|
|
9
9
|
import type { AdminContext } from '../ctx.js';
|
|
10
|
-
import type { DataSourceJob, DataSourceJobGetResult, DataSourceJobResult, DataSourceJobsListResult, DataSourceJobSource } from '../types/dataSourceJobs.js';
|
|
10
|
+
import type { DataSourceAddManyResult, DataSourceJob, DataSourceJobGetResult, DataSourceJobResult, DataSourceJobsListResult, DataSourceJobSource } from '../types/dataSourceJobs.js';
|
|
11
11
|
/** @internal Consumed by the CLI skin. */
|
|
12
12
|
export declare const DEFAULT_WAIT_TIMEOUT_MS: number;
|
|
13
13
|
export interface StartParams {
|
|
@@ -54,6 +54,57 @@ export interface UploadManifestParams {
|
|
|
54
54
|
export declare function uploadManifest(ctx: AdminContext, params: UploadManifestParams): Promise<{
|
|
55
55
|
key: string;
|
|
56
56
|
}>;
|
|
57
|
+
/** One document of `addMany`: what a mapper's `documents()` returns. */
|
|
58
|
+
export interface AddManyDocument {
|
|
59
|
+
/**
|
|
60
|
+
* The document's identity. Giving one the source already holds supersedes
|
|
61
|
+
* that document.
|
|
62
|
+
*/
|
|
63
|
+
externalId: string;
|
|
64
|
+
title: string;
|
|
65
|
+
/** The document's text. Chunked and embedded as markdown. */
|
|
66
|
+
markdown: string;
|
|
67
|
+
/** Scalars only, up to 16 keys; filterable at search time. */
|
|
68
|
+
metadata?: Record<string, string | number | boolean>;
|
|
69
|
+
/** What the document is about, as your own ids. */
|
|
70
|
+
entities?: string[];
|
|
71
|
+
/** External ids this document replaces. */
|
|
72
|
+
replaces?: string | string[];
|
|
73
|
+
}
|
|
74
|
+
export interface AddManyParams {
|
|
75
|
+
/** Data source slug. Created on first use, like `add`. */
|
|
76
|
+
slug: string;
|
|
77
|
+
documents: Iterable<AddManyDocument> | AsyncIterable<AddManyDocument>;
|
|
78
|
+
/** Pause when estimated spend reaches this; default 1.5× the projection. */
|
|
79
|
+
budgetDollars?: number;
|
|
80
|
+
/** Called after each part is stored, with how many documents are stored so far. */
|
|
81
|
+
onProgress?: (stored: number) => void;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Add many documents at once, built as one job: records rendered to text
|
|
85
|
+
* (a row of the app's own table as a markdown card), in any number. They are
|
|
86
|
+
* stored a thousand at a time, then one job registers and builds them. The
|
|
87
|
+
* job plans, and starts on its own once its plan passes the gates, within
|
|
88
|
+
* `budgetDollars`. Returns once the job has started; follow it with `get` or
|
|
89
|
+
* `waitForJob`.
|
|
90
|
+
*
|
|
91
|
+
* An external id the source already holds is superseded when the job
|
|
92
|
+
* finishes; unchanged markdown builds nothing, and new metadata or entities
|
|
93
|
+
* on unchanged markdown update in place. A document that fails validation is
|
|
94
|
+
* left out and counted in the job's quarantine; the rest go in. Nothing is
|
|
95
|
+
* removed for being left out of a later call.
|
|
96
|
+
*
|
|
97
|
+
* @throws AdminApiError `data_source_mapped` (422) — the source has a mapper,
|
|
98
|
+
* which is what makes its documents; `data_source_busy` (422) — a move, a
|
|
99
|
+
* candidate version or another job is in flight; `invalid_parts` (400);
|
|
100
|
+
* `add_many_not_found` (404) — the upload took longer than two hours.
|
|
101
|
+
* @example
|
|
102
|
+
* const { job } = await admin.dataSourceJobs.addMany({
|
|
103
|
+
* slug: 'claims',
|
|
104
|
+
* documents: [{ externalId: 'c-1', title: 'Claim 1', markdown: '…' }],
|
|
105
|
+
* });
|
|
106
|
+
*/
|
|
107
|
+
export declare function addMany(ctx: AdminContext, params: AddManyParams): Promise<DataSourceAddManyResult>;
|
|
57
108
|
export interface ListParams {
|
|
58
109
|
/** Only this source's jobs; omit for every job on the app. */
|
|
59
110
|
slug?: string;
|
|
@@ -97,8 +97,9 @@ export interface AddDocumentResult {
|
|
|
97
97
|
*
|
|
98
98
|
* @throws AdminApiError `invalid_content_hash` (400) — malformed hash
|
|
99
99
|
* (internal; `addDocument` computes this from `content`).
|
|
100
|
-
* @throws AdminApiError `
|
|
101
|
-
*
|
|
100
|
+
* @throws AdminApiError `data_source_full` (422) — the source holds what its
|
|
101
|
+
* placement does: the shared pool's chunks per source, or its dedicated
|
|
102
|
+
* capacity's room. The message says which, and the way out.
|
|
102
103
|
* @example
|
|
103
104
|
* const result = await admin.dataSources.addDocument({
|
|
104
105
|
* slug: 'policies',
|
|
@@ -217,8 +218,7 @@ export declare function documents(ctx: AdminContext, params: DocumentsParams): P
|
|
|
217
218
|
*/
|
|
218
219
|
export declare function allDocuments(ctx: AdminContext, params: Omit<DocumentsParams, 'cursor' | 'limit'>): Promise<DataSourcesDocumentStatus[]>;
|
|
219
220
|
/**
|
|
220
|
-
* List all data sources with document counts,
|
|
221
|
-
* hold (`documentLimit`, null when unlimited), build progress, and active
|
|
221
|
+
* List all data sources with document counts, build progress, and active
|
|
222
222
|
* version metadata.
|
|
223
223
|
*
|
|
224
224
|
* @example
|
package/dist/prod.js
CHANGED
|
@@ -1345,7 +1345,8 @@ async function releasesWait(ctx, a) {
|
|
|
1345
1345
|
);
|
|
1346
1346
|
}
|
|
1347
1347
|
const commit = expandCommitSha(rawCommit);
|
|
1348
|
-
const
|
|
1348
|
+
const timeoutSeconds = a.num("timeout");
|
|
1349
|
+
const timeoutMs = timeoutSeconds === void 0 ? void 0 : timeoutSeconds * 1e3;
|
|
1349
1350
|
const result = await waitForCommit(ctx, {
|
|
1350
1351
|
commitSha: commit,
|
|
1351
1352
|
timeoutMs,
|
|
@@ -4418,6 +4419,7 @@ function link(ctx, params) {
|
|
|
4418
4419
|
}
|
|
4419
4420
|
|
|
4420
4421
|
// src/ops/dataSourceJobs.ts
|
|
4422
|
+
import { gzipSync } from "node:zlib";
|
|
4421
4423
|
var DEFAULT_WAIT_TIMEOUT_MS2 = 60 * 60 * 1e3;
|
|
4422
4424
|
var POLL_MS3 = 5e3;
|
|
4423
4425
|
function base2(appId) {
|
|
@@ -4445,6 +4447,66 @@ async function uploadManifest(ctx, params) {
|
|
|
4445
4447
|
await uploadDirect(token.upload, params.content, "manifest.jsonl");
|
|
4446
4448
|
return { key: token.key };
|
|
4447
4449
|
}
|
|
4450
|
+
var PART_DOCUMENTS = 1e3;
|
|
4451
|
+
var PART_BYTES = 8 * 1024 * 1024;
|
|
4452
|
+
async function addMany(ctx, params) {
|
|
4453
|
+
const addManyBase = `/_internal/v2/apps/${ctx.appId}/datasources/add-many`;
|
|
4454
|
+
const { uploadId } = await call(
|
|
4455
|
+
ctx,
|
|
4456
|
+
"POST",
|
|
4457
|
+
`${addManyBase}/open`,
|
|
4458
|
+
{ slug: params.slug }
|
|
4459
|
+
);
|
|
4460
|
+
const parts = [];
|
|
4461
|
+
let lines = [];
|
|
4462
|
+
let bytes = 0;
|
|
4463
|
+
let documentCount = 0;
|
|
4464
|
+
const storeLines = async () => {
|
|
4465
|
+
if (lines.length === 0) {
|
|
4466
|
+
return;
|
|
4467
|
+
}
|
|
4468
|
+
const grant = await call(
|
|
4469
|
+
ctx,
|
|
4470
|
+
"POST",
|
|
4471
|
+
`${addManyBase}/part`,
|
|
4472
|
+
{ slug: params.slug, uploadId }
|
|
4473
|
+
);
|
|
4474
|
+
await uploadDirect(
|
|
4475
|
+
grant,
|
|
4476
|
+
gzipSync(Buffer.from(lines.join("\n") + "\n")),
|
|
4477
|
+
`${grant.part}.ndjson.gz`
|
|
4478
|
+
);
|
|
4479
|
+
parts.push({ part: grant.part, count: lines.length });
|
|
4480
|
+
lines = [];
|
|
4481
|
+
bytes = 0;
|
|
4482
|
+
params.onProgress?.(documentCount);
|
|
4483
|
+
};
|
|
4484
|
+
for await (const d of params.documents) {
|
|
4485
|
+
const line = JSON.stringify({ o: 0, i: documentCount, d });
|
|
4486
|
+
lines.push(line);
|
|
4487
|
+
bytes += line.length;
|
|
4488
|
+
documentCount += 1;
|
|
4489
|
+
if (lines.length >= PART_DOCUMENTS || bytes >= PART_BYTES) {
|
|
4490
|
+
await storeLines();
|
|
4491
|
+
}
|
|
4492
|
+
}
|
|
4493
|
+
await storeLines();
|
|
4494
|
+
if (parts.length === 0) {
|
|
4495
|
+
return { documentCount: 0, job: null };
|
|
4496
|
+
}
|
|
4497
|
+
const { job } = await call(
|
|
4498
|
+
ctx,
|
|
4499
|
+
"POST",
|
|
4500
|
+
`${addManyBase}/start`,
|
|
4501
|
+
{
|
|
4502
|
+
slug: params.slug,
|
|
4503
|
+
uploadId,
|
|
4504
|
+
parts,
|
|
4505
|
+
...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {}
|
|
4506
|
+
}
|
|
4507
|
+
);
|
|
4508
|
+
return { documentCount, job };
|
|
4509
|
+
}
|
|
4448
4510
|
function list11(ctx, params) {
|
|
4449
4511
|
return call(
|
|
4450
4512
|
ctx,
|
|
@@ -5763,6 +5825,16 @@ var dataSourceJobsSpecs = {
|
|
|
5763
5825
|
message: "Give the job something to read: --store <name> [--prefix <p>], or --manifest <file.jsonl>."
|
|
5764
5826
|
}
|
|
5765
5827
|
},
|
|
5828
|
+
"datasources add-many": {
|
|
5829
|
+
usage: "Usage: remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>",
|
|
5830
|
+
positionals: [{ name: "file", required: true }],
|
|
5831
|
+
flags: {
|
|
5832
|
+
source: { type: "string" },
|
|
5833
|
+
budget: { type: "number", min: 0.01 },
|
|
5834
|
+
wait: { type: "boolean" },
|
|
5835
|
+
timeout: { type: "string" }
|
|
5836
|
+
}
|
|
5837
|
+
},
|
|
5766
5838
|
"datasources jobs list": {
|
|
5767
5839
|
usage: "Usage: remy-admin datasources jobs list [--source <slug>]",
|
|
5768
5840
|
flags: { source: { type: "string" } }
|
|
@@ -5866,6 +5938,65 @@ async function jobsStart(ctx, a) {
|
|
|
5866
5938
|
...job.autoApprove ? {} : { note: "Review the plan, then `datasources jobs approve <id>`." }
|
|
5867
5939
|
});
|
|
5868
5940
|
}
|
|
5941
|
+
function readDocuments(file) {
|
|
5942
|
+
const abs = path5.isAbsolute(file) ? file : path5.join(WORKSPACE_DIR, file);
|
|
5943
|
+
let content;
|
|
5944
|
+
try {
|
|
5945
|
+
content = fs5.readFileSync(abs, "utf8");
|
|
5946
|
+
} catch (err) {
|
|
5947
|
+
fatal(`Could not read "${file}": ${err.message}`);
|
|
5948
|
+
}
|
|
5949
|
+
const documents2 = [];
|
|
5950
|
+
content.split("\n").forEach((line, i) => {
|
|
5951
|
+
const trimmed = line.trim();
|
|
5952
|
+
if (!trimmed) {
|
|
5953
|
+
return;
|
|
5954
|
+
}
|
|
5955
|
+
let parsed;
|
|
5956
|
+
try {
|
|
5957
|
+
parsed = JSON.parse(trimmed);
|
|
5958
|
+
} catch {
|
|
5959
|
+
fatal(`${file}:${i + 1} is not valid JSON.`);
|
|
5960
|
+
}
|
|
5961
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
5962
|
+
fatal(
|
|
5963
|
+
`${file}:${i + 1} needs { "externalId": "...", "title": "...", "markdown": "..." }.`
|
|
5964
|
+
);
|
|
5965
|
+
}
|
|
5966
|
+
documents2.push(parsed);
|
|
5967
|
+
});
|
|
5968
|
+
if (documents2.length === 0) {
|
|
5969
|
+
fatal(`"${file}" has no documents.`);
|
|
5970
|
+
}
|
|
5971
|
+
return documents2;
|
|
5972
|
+
}
|
|
5973
|
+
async function addMany2(ctx, a) {
|
|
5974
|
+
const slug = sourceOf(a);
|
|
5975
|
+
const documents2 = readDocuments(a.required("file"));
|
|
5976
|
+
const budget = a.num("budget");
|
|
5977
|
+
const { documentCount, job } = await addMany(ctx, {
|
|
5978
|
+
slug,
|
|
5979
|
+
documents: documents2,
|
|
5980
|
+
...budget !== void 0 ? { budgetDollars: budget } : {},
|
|
5981
|
+
onProgress: (stored) => progress(
|
|
5982
|
+
`stored ${stored.toLocaleString()}/${documents2.length.toLocaleString()}\u2026`
|
|
5983
|
+
)
|
|
5984
|
+
});
|
|
5985
|
+
if (!job || !a.bool("wait")) {
|
|
5986
|
+
printJson({
|
|
5987
|
+
dataSource: slug,
|
|
5988
|
+
documentCount,
|
|
5989
|
+
job,
|
|
5990
|
+
note: "Planning, then running once the plan passes the gates. Follow it with `datasources jobs status <id> --wait`."
|
|
5991
|
+
});
|
|
5992
|
+
return;
|
|
5993
|
+
}
|
|
5994
|
+
await waitAndReport(ctx, a, job.id, "job", {
|
|
5995
|
+
dataSource: slug,
|
|
5996
|
+
action: "add-many",
|
|
5997
|
+
documentCount
|
|
5998
|
+
});
|
|
5999
|
+
}
|
|
5869
6000
|
async function jobsList(ctx, a) {
|
|
5870
6001
|
const slug = a.str("source");
|
|
5871
6002
|
printJson(await list11(ctx, slug ? { slug } : void 0));
|
|
@@ -5926,6 +6057,7 @@ async function jobsCancel(ctx, a) {
|
|
|
5926
6057
|
}
|
|
5927
6058
|
var dataSourceJobsHandlers = {
|
|
5928
6059
|
"datasources jobs start": jobsStart,
|
|
6060
|
+
"datasources add-many": addMany2,
|
|
5929
6061
|
"datasources jobs list": jobsList,
|
|
5930
6062
|
"datasources jobs status": jobsStatus,
|
|
5931
6063
|
"datasources jobs approve": jobsApprove,
|
|
@@ -5993,6 +6125,27 @@ Bulk ingestion (jobs):
|
|
|
5993
6125
|
a candidate version exists, and moves, re-vectorizes and deletes refuse while
|
|
5994
6126
|
a job is in flight (data_source_busy). Interactive \`add\` keeps working.
|
|
5995
6127
|
|
|
6128
|
+
Adding many documents (add-many):
|
|
6129
|
+
remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>
|
|
6130
|
+
|
|
6131
|
+
For records rendered to text, in any number: JSONL, one document per line,
|
|
6132
|
+
{"externalId": "claim-123", "title": "\u2026", "markdown": "\u2026", "metadata": {\u2026},
|
|
6133
|
+
"entities": ["\u2026"], "replaces": ["\u2026"]}. externalId, title and markdown are
|
|
6134
|
+
required. The file is stored a thousand documents at a time, then one job
|
|
6135
|
+
registers and builds them: it plans, then runs on its own once the plan
|
|
6136
|
+
passes the gates (the two \`jobs start\` has), within --budget. Follow it
|
|
6137
|
+
with \`jobs status <id>\`, or pass --wait.
|
|
6138
|
+
|
|
6139
|
+
externalId is a document's identity. One the source already holds is
|
|
6140
|
+
superseded when the job finishes; unchanged markdown builds nothing; new
|
|
6141
|
+
metadata or entities on unchanged markdown update in place. A document that
|
|
6142
|
+
fails validation is left out and counted in the job's quarantine
|
|
6143
|
+
(\`jobs quarantine <id>\`), named by its place in the file from 0; the
|
|
6144
|
+
rest go in. Nothing is removed for being left out of a later file.
|
|
6145
|
+
|
|
6146
|
+
Not on a source with a mapper (data_source_mapped): there the mapper makes
|
|
6147
|
+
the documents. One job per source at a time (data_source_busy).
|
|
6148
|
+
|
|
5996
6149
|
--wait exit codes: 0 done \xB7 1 the job failed, paused, was cancelled, or sits
|
|
5997
6150
|
in planned because a gate refused it (approve once the cause is fixed) \xB7
|
|
5998
6151
|
2 still running when --timeout elapsed (default 3600s).
|
|
@@ -7316,6 +7469,7 @@ Subcommands:
|
|
|
7316
7469
|
create Create an empty source, optionally on dedicated capacity
|
|
7317
7470
|
config Show or change how a corpus is processed and searched
|
|
7318
7471
|
move Move a corpus between shared and dedicated capacity, data intact
|
|
7472
|
+
add-many Add a file of documents (JSONL records rendered to text), built as one job
|
|
7319
7473
|
jobs Bulk-load a corpus from a file store or a manifest, with a plan first
|
|
7320
7474
|
connect Follow an S3 bucket the customer owns (credentials = app secret names)
|
|
7321
7475
|
sync Bring a connected source up to date with its bucket
|
|
@@ -7348,6 +7502,7 @@ Usage:
|
|
|
7348
7502
|
remy-admin datasources create --source <slug> [--name <name>] [--placement <resource-id|shared>] [rebuild settings...]
|
|
7349
7503
|
remy-admin datasources config [--source <slug>] [--placement <resource-id|shared>] [settings...]
|
|
7350
7504
|
remy-admin datasources move [--source <slug>] --to <resource-id|shared> [--chains <n>] [--restart] [--wait] [--timeout <sec>]
|
|
7505
|
+
remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>
|
|
7351
7506
|
remy-admin datasources jobs start|list|status|approve|pause|resume|retry|cancel \u2026 (see below)
|
|
7352
7507
|
remy-admin datasources connect|sync|connector|disconnect \u2026 (see below)
|
|
7353
7508
|
remy-admin datasources hydrate|reindex|sample \u2026 (see below)
|
|
@@ -7485,9 +7640,12 @@ Notes:
|
|
|
7485
7640
|
Exits ${EXIT.buildFailed} if a document failed, ${EXIT.timeout} on timeout.
|
|
7486
7641
|
Re-adding an unchanged file is free: no upload, no re-embedding. Re-adding
|
|
7487
7642
|
with a different --metadata updates the tags in place, also free.
|
|
7488
|
-
add
|
|
7489
|
-
|
|
7490
|
-
(
|
|
7643
|
+
add is refused with data_source_full (422) once a source holds what its
|
|
7644
|
+
placement does: a shared-pool source at the per-source cap moves to
|
|
7645
|
+
dedicated capacity (\`move\`), a dedicated one is resized. The message
|
|
7646
|
+
says which.
|
|
7647
|
+
add is for a file at a time. For records rendered to text in any number,
|
|
7648
|
+
use add-many; for files in a store or behind URLs, a job.
|
|
7491
7649
|
--metadata tags documents for search-time filtering (scalars, \u226416 keys).
|
|
7492
7650
|
--entities names what a document is about, as your own ids (\u2264256, no spaces
|
|
7493
7651
|
or commas); an empty --entities clears them. Also free on a re-add.
|
|
@@ -7527,6 +7685,9 @@ Examples:
|
|
|
7527
7685
|
remy-admin datasources jobs start --source archive --store raw --prefix 2024/ --wait
|
|
7528
7686
|
remy-admin datasources jobs approve <id> --wait
|
|
7529
7687
|
|
|
7688
|
+
# Add records rendered to text, in any number: one job, priced first
|
|
7689
|
+
remy-admin datasources add-many --source claims --wait claims.jsonl
|
|
7690
|
+
|
|
7530
7691
|
# Follow a customer's bucket: secrets by name, first sync stops for approval
|
|
7531
7692
|
remy-admin secrets set ARCHIVE_S3_KEY --prod <value> # never on a command line you share
|
|
7532
7693
|
remy-admin datasources connect --source archive --bucket acme-docs --region us-east-1 --prefix contracts/ --access-key-secret ARCHIVE_S3_KEY --secret-key-secret ARCHIVE_S3_SECRET --budget-per-sync 5
|
|
@@ -8473,7 +8634,6 @@ async function jewelsDryrun(ctx, a) {
|
|
|
8473
8634
|
);
|
|
8474
8635
|
}
|
|
8475
8636
|
var TRAIN_POLL_MS = 15e3;
|
|
8476
|
-
var TRAIN_WAIT_LIMIT_MS = 45 * 6e4;
|
|
8477
8637
|
async function jewelsTrain(ctx, a) {
|
|
8478
8638
|
const started = await train(ctx, { methodId: a.required("methodId") });
|
|
8479
8639
|
if (!a.bool("wait")) {
|
|
@@ -8482,9 +8642,8 @@ async function jewelsTrain(ctx, a) {
|
|
|
8482
8642
|
}
|
|
8483
8643
|
printJson({ runId: started.run?.id, summary: started.summary });
|
|
8484
8644
|
const runId = started.run?.id;
|
|
8485
|
-
const deadline = Date.now() + TRAIN_WAIT_LIMIT_MS;
|
|
8486
8645
|
let lastLogLen = 0;
|
|
8487
|
-
while (
|
|
8646
|
+
while (true) {
|
|
8488
8647
|
await sleep(TRAIN_POLL_MS);
|
|
8489
8648
|
const { run: run3 } = await getRun2(ctx, runId);
|
|
8490
8649
|
const log = Array.isArray(run3.log) ? run3.log : [];
|
|
@@ -8513,9 +8672,6 @@ async function jewelsTrain(ctx, a) {
|
|
|
8513
8672
|
return;
|
|
8514
8673
|
}
|
|
8515
8674
|
}
|
|
8516
|
-
fatal(
|
|
8517
|
-
`Timed out waiting for run ${runId} (still in flight; check 'jewels run ${runId}').`
|
|
8518
|
-
);
|
|
8519
8675
|
}
|
|
8520
8676
|
function compactRunLog(run3) {
|
|
8521
8677
|
let out = { ...run3 };
|
|
@@ -54,6 +54,10 @@ export type DataSourceJobSource = {
|
|
|
54
54
|
*/
|
|
55
55
|
type: 'object';
|
|
56
56
|
ref: string;
|
|
57
|
+
} | {
|
|
58
|
+
/** The documents a `datasources add-many` (or an app's `addMany()`) gave, and how many. */
|
|
59
|
+
type: 'documents';
|
|
60
|
+
documents: number;
|
|
57
61
|
};
|
|
58
62
|
/**
|
|
59
63
|
* planning → planned → running ⇄ paused → done | failed | cancelled.
|
|
@@ -272,6 +276,25 @@ export interface DataSourceJobGetResult {
|
|
|
272
276
|
export interface DataSourceJobResult {
|
|
273
277
|
job: DataSourceJob;
|
|
274
278
|
}
|
|
279
|
+
/** POST /datasources/add-many/open */
|
|
280
|
+
export interface DataSourceAddManyOpenResult {
|
|
281
|
+
/** Names the upload on `add-many/part` and `add-many/start`. */
|
|
282
|
+
uploadId: string;
|
|
283
|
+
}
|
|
284
|
+
/** POST /datasources/add-many/part */
|
|
285
|
+
export interface DataSourceAddManyPartResult {
|
|
286
|
+
/** The part's number, which `add-many/start` lists it by. */
|
|
287
|
+
part: number;
|
|
288
|
+
uploadUrl: string;
|
|
289
|
+
uploadFields: Record<string, string>;
|
|
290
|
+
}
|
|
291
|
+
/** What `addMany` answers with. */
|
|
292
|
+
export interface DataSourceAddManyResult {
|
|
293
|
+
/** How many documents were given. */
|
|
294
|
+
documentCount: number;
|
|
295
|
+
/** The job registering and building them; null only when none were given. */
|
|
296
|
+
job: DataSourceJob | null;
|
|
297
|
+
}
|
|
275
298
|
/** POST /datasources/jobs/manifest-token */
|
|
276
299
|
export interface DataSourceJobManifestTokenResult {
|
|
277
300
|
key: string;
|
|
@@ -352,11 +352,6 @@ export interface DataSourcesListEntry {
|
|
|
352
352
|
name: string | null;
|
|
353
353
|
/** Total document count (active pipeline). */
|
|
354
354
|
documentCount: number;
|
|
355
|
-
/**
|
|
356
|
-
* The most documents the source may hold, the app's limit; null when
|
|
357
|
-
* unlimited. An add past it is refused with `document_limit_reached`.
|
|
358
|
-
*/
|
|
359
|
-
documentLimit: number | null;
|
|
360
355
|
counts: DataSourcesBuildProgress;
|
|
361
356
|
/** Source-document bytes (does not include vector storage). */
|
|
362
357
|
storageBytes: number;
|