@madewithremy/admin 0.1.67 → 0.1.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/index.js +62 -0
- package/dist/index.js.map +2 -2
- package/dist/ops/dataSourceJobs.d.ts +52 -1
- package/dist/ops/dataSources.d.ts +4 -4
- package/dist/ops/db.d.ts +8 -3
- package/dist/prod.js +167 -3
- package/dist/types/dataSourceJobs.d.ts +23 -0
- package/dist/types/dataSources.d.ts +0 -5
- package/package.json +1 -1
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* commands/dataSourceJobs.ts owns flags, progress printing and exit codes.
|
|
8
8
|
*/
|
|
9
9
|
import type { AdminContext } from '../ctx.js';
|
|
10
|
-
import type { DataSourceJob, DataSourceJobGetResult, DataSourceJobResult, DataSourceJobsListResult, DataSourceJobSource } from '../types/dataSourceJobs.js';
|
|
10
|
+
import type { DataSourceAddManyResult, DataSourceJob, DataSourceJobGetResult, DataSourceJobResult, DataSourceJobsListResult, DataSourceJobSource } from '../types/dataSourceJobs.js';
|
|
11
11
|
/** @internal Consumed by the CLI skin. */
|
|
12
12
|
export declare const DEFAULT_WAIT_TIMEOUT_MS: number;
|
|
13
13
|
export interface StartParams {
|
|
@@ -54,6 +54,57 @@ export interface UploadManifestParams {
|
|
|
54
54
|
export declare function uploadManifest(ctx: AdminContext, params: UploadManifestParams): Promise<{
|
|
55
55
|
key: string;
|
|
56
56
|
}>;
|
|
57
|
+
/** One document of `addMany`: what a mapper's `documents()` returns. */
|
|
58
|
+
export interface AddManyDocument {
|
|
59
|
+
/**
|
|
60
|
+
* The document's identity. Giving one the source already holds supersedes
|
|
61
|
+
* that document.
|
|
62
|
+
*/
|
|
63
|
+
externalId: string;
|
|
64
|
+
title: string;
|
|
65
|
+
/** The document's text. Chunked and embedded as markdown. */
|
|
66
|
+
markdown: string;
|
|
67
|
+
/** Scalars only, up to 16 keys; filterable at search time. */
|
|
68
|
+
metadata?: Record<string, string | number | boolean>;
|
|
69
|
+
/** What the document is about, as your own ids. */
|
|
70
|
+
entities?: string[];
|
|
71
|
+
/** External ids this document replaces. */
|
|
72
|
+
replaces?: string | string[];
|
|
73
|
+
}
|
|
74
|
+
export interface AddManyParams {
|
|
75
|
+
/** Data source slug. Created on first use, like `add`. */
|
|
76
|
+
slug: string;
|
|
77
|
+
documents: Iterable<AddManyDocument> | AsyncIterable<AddManyDocument>;
|
|
78
|
+
/** Pause when estimated spend reaches this; default 1.5× the projection. */
|
|
79
|
+
budgetDollars?: number;
|
|
80
|
+
/** Called after each part is stored, with how many documents are stored so far. */
|
|
81
|
+
onProgress?: (stored: number) => void;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Add many documents at once, built as one job: records rendered to text
|
|
85
|
+
* (a row of the app's own table as a markdown card), in any number. They are
|
|
86
|
+
* stored a thousand at a time, then one job registers and builds them. The
|
|
87
|
+
* job plans, and starts on its own once its plan passes the gates, within
|
|
88
|
+
* `budgetDollars`. Returns once the job has started; follow it with `get` or
|
|
89
|
+
* `waitForJob`.
|
|
90
|
+
*
|
|
91
|
+
* An external id the source already holds is superseded when the job
|
|
92
|
+
* finishes; unchanged markdown builds nothing, and new metadata or entities
|
|
93
|
+
* on unchanged markdown update in place. A document that fails validation is
|
|
94
|
+
* left out and counted in the job's quarantine; the rest go in. Nothing is
|
|
95
|
+
* removed for being left out of a later call.
|
|
96
|
+
*
|
|
97
|
+
* @throws AdminApiError `data_source_mapped` (422) — the source has a mapper,
|
|
98
|
+
* which is what makes its documents; `data_source_busy` (422) — a move, a
|
|
99
|
+
* candidate version or another job is in flight; `invalid_parts` (400);
|
|
100
|
+
* `add_many_not_found` (404) — the upload took longer than two hours.
|
|
101
|
+
* @example
|
|
102
|
+
* const { job } = await admin.dataSourceJobs.addMany({
|
|
103
|
+
* slug: 'claims',
|
|
104
|
+
* documents: [{ externalId: 'c-1', title: 'Claim 1', markdown: '…' }],
|
|
105
|
+
* });
|
|
106
|
+
*/
|
|
107
|
+
export declare function addMany(ctx: AdminContext, params: AddManyParams): Promise<DataSourceAddManyResult>;
|
|
57
108
|
export interface ListParams {
|
|
58
109
|
/** Only this source's jobs; omit for every job on the app. */
|
|
59
110
|
slug?: string;
|
|
@@ -97,8 +97,9 @@ export interface AddDocumentResult {
|
|
|
97
97
|
*
|
|
98
98
|
* @throws AdminApiError `invalid_content_hash` (400) — malformed hash
|
|
99
99
|
* (internal; `addDocument` computes this from `content`).
|
|
100
|
-
* @throws AdminApiError `
|
|
101
|
-
*
|
|
100
|
+
* @throws AdminApiError `data_source_full` (422) — the source holds what its
|
|
101
|
+
* placement does: the shared pool's chunks per source, or its dedicated
|
|
102
|
+
* capacity's room. The message says which, and the way out.
|
|
102
103
|
* @example
|
|
103
104
|
* const result = await admin.dataSources.addDocument({
|
|
104
105
|
* slug: 'policies',
|
|
@@ -217,8 +218,7 @@ export declare function documents(ctx: AdminContext, params: DocumentsParams): P
|
|
|
217
218
|
*/
|
|
218
219
|
export declare function allDocuments(ctx: AdminContext, params: Omit<DocumentsParams, 'cursor' | 'limit'>): Promise<DataSourcesDocumentStatus[]>;
|
|
219
220
|
/**
|
|
220
|
-
* List all data sources with document counts,
|
|
221
|
-
* hold (`documentLimit`, null when unlimited), build progress, and active
|
|
221
|
+
* List all data sources with document counts, build progress, and active
|
|
222
222
|
* version metadata.
|
|
223
223
|
*
|
|
224
224
|
* @example
|
package/dist/ops/db.d.ts
CHANGED
|
@@ -19,6 +19,10 @@ import type { DbHistoryPurgeResult, DbHistoryResult, DbQueryResult } from '../ty
|
|
|
19
19
|
* too large) throws with `errorString: 'sql_error'`, and the body's
|
|
20
20
|
* `errorMessage` is the database's own message.
|
|
21
21
|
*
|
|
22
|
+
* A statement that changes live is recorded in the workspace's audit log:
|
|
23
|
+
* who ran it and when, which database and how many rows changed, never the
|
|
24
|
+
* SQL.
|
|
25
|
+
*
|
|
22
26
|
* @param sql The SQL statement to execute.
|
|
23
27
|
* @throws AdminApiError `no_live_release` (404) — the app has not been
|
|
24
28
|
* deployed; `no_database` (404) — the live release has no database;
|
|
@@ -70,9 +74,10 @@ export declare function historyQuery(ctx: AdminContext, releaseId: string, sql:
|
|
|
70
74
|
/**
|
|
71
75
|
* Run one SQL statement against live with a past release's database attached
|
|
72
76
|
* as the schema `history`: `history.<table>` is the past table, and an
|
|
73
|
-
* unqualified name is live's. Writes to live are real, as with `query`,
|
|
74
|
-
*
|
|
75
|
-
*
|
|
77
|
+
* unqualified name is live's. Writes to live are real, as with `query`, run
|
|
78
|
+
* in one transaction, and are recorded in the workspace's audit log as
|
|
79
|
+
* `query`'s are; the attached copy is private to the call, so nothing
|
|
80
|
+
* written into it lasts.
|
|
76
81
|
*
|
|
77
82
|
* A past database larger than 1 GB can't be attached: read it with
|
|
78
83
|
* `historyQuery` instead, and write what's needed to live with `query`.
|
package/dist/prod.js
CHANGED
|
@@ -2457,6 +2457,10 @@ Notes:
|
|
|
2457
2457
|
SQL don't parse flags, so a statement may safely begin with a '--'
|
|
2458
2458
|
comment. Always quote the SQL.
|
|
2459
2459
|
- The history commands use the database named as live's default.
|
|
2460
|
+
- A 'db query' or 'db history attach' that changes live is recorded in the
|
|
2461
|
+
workspace's audit log, as a purge is: who ran it and when, which database
|
|
2462
|
+
and how many rows changed, never the SQL. Workspace admins read the log
|
|
2463
|
+
in Workspace Settings.
|
|
2460
2464
|
- The first history command on a copy fetches it, so it takes longer for a
|
|
2461
2465
|
large database; the ones after it are quick.
|
|
2462
2466
|
- Errors: release_not_found (no release of this app that was ever live has
|
|
@@ -4419,6 +4423,7 @@ function link(ctx, params) {
|
|
|
4419
4423
|
}
|
|
4420
4424
|
|
|
4421
4425
|
// src/ops/dataSourceJobs.ts
|
|
4426
|
+
import { gzipSync } from "node:zlib";
|
|
4422
4427
|
var DEFAULT_WAIT_TIMEOUT_MS2 = 60 * 60 * 1e3;
|
|
4423
4428
|
var POLL_MS3 = 5e3;
|
|
4424
4429
|
function base2(appId) {
|
|
@@ -4446,6 +4451,66 @@ async function uploadManifest(ctx, params) {
|
|
|
4446
4451
|
await uploadDirect(token.upload, params.content, "manifest.jsonl");
|
|
4447
4452
|
return { key: token.key };
|
|
4448
4453
|
}
|
|
4454
|
+
var PART_DOCUMENTS = 1e3;
|
|
4455
|
+
var PART_BYTES = 8 * 1024 * 1024;
|
|
4456
|
+
async function addMany(ctx, params) {
|
|
4457
|
+
const addManyBase = `/_internal/v2/apps/${ctx.appId}/datasources/add-many`;
|
|
4458
|
+
const { uploadId } = await call(
|
|
4459
|
+
ctx,
|
|
4460
|
+
"POST",
|
|
4461
|
+
`${addManyBase}/open`,
|
|
4462
|
+
{ slug: params.slug }
|
|
4463
|
+
);
|
|
4464
|
+
const parts = [];
|
|
4465
|
+
let lines = [];
|
|
4466
|
+
let bytes = 0;
|
|
4467
|
+
let documentCount = 0;
|
|
4468
|
+
const storeLines = async () => {
|
|
4469
|
+
if (lines.length === 0) {
|
|
4470
|
+
return;
|
|
4471
|
+
}
|
|
4472
|
+
const grant = await call(
|
|
4473
|
+
ctx,
|
|
4474
|
+
"POST",
|
|
4475
|
+
`${addManyBase}/part`,
|
|
4476
|
+
{ slug: params.slug, uploadId }
|
|
4477
|
+
);
|
|
4478
|
+
await uploadDirect(
|
|
4479
|
+
grant,
|
|
4480
|
+
gzipSync(Buffer.from(lines.join("\n") + "\n")),
|
|
4481
|
+
`${grant.part}.ndjson.gz`
|
|
4482
|
+
);
|
|
4483
|
+
parts.push({ part: grant.part, count: lines.length });
|
|
4484
|
+
lines = [];
|
|
4485
|
+
bytes = 0;
|
|
4486
|
+
params.onProgress?.(documentCount);
|
|
4487
|
+
};
|
|
4488
|
+
for await (const d of params.documents) {
|
|
4489
|
+
const line = JSON.stringify({ o: 0, i: documentCount, d });
|
|
4490
|
+
lines.push(line);
|
|
4491
|
+
bytes += line.length;
|
|
4492
|
+
documentCount += 1;
|
|
4493
|
+
if (lines.length >= PART_DOCUMENTS || bytes >= PART_BYTES) {
|
|
4494
|
+
await storeLines();
|
|
4495
|
+
}
|
|
4496
|
+
}
|
|
4497
|
+
await storeLines();
|
|
4498
|
+
if (parts.length === 0) {
|
|
4499
|
+
return { documentCount: 0, job: null };
|
|
4500
|
+
}
|
|
4501
|
+
const { job } = await call(
|
|
4502
|
+
ctx,
|
|
4503
|
+
"POST",
|
|
4504
|
+
`${addManyBase}/start`,
|
|
4505
|
+
{
|
|
4506
|
+
slug: params.slug,
|
|
4507
|
+
uploadId,
|
|
4508
|
+
parts,
|
|
4509
|
+
...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {}
|
|
4510
|
+
}
|
|
4511
|
+
);
|
|
4512
|
+
return { documentCount, job };
|
|
4513
|
+
}
|
|
4449
4514
|
function list11(ctx, params) {
|
|
4450
4515
|
return call(
|
|
4451
4516
|
ctx,
|
|
@@ -5764,6 +5829,16 @@ var dataSourceJobsSpecs = {
|
|
|
5764
5829
|
message: "Give the job something to read: --store <name> [--prefix <p>], or --manifest <file.jsonl>."
|
|
5765
5830
|
}
|
|
5766
5831
|
},
|
|
5832
|
+
"datasources add-many": {
|
|
5833
|
+
usage: "Usage: remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>",
|
|
5834
|
+
positionals: [{ name: "file", required: true }],
|
|
5835
|
+
flags: {
|
|
5836
|
+
source: { type: "string" },
|
|
5837
|
+
budget: { type: "number", min: 0.01 },
|
|
5838
|
+
wait: { type: "boolean" },
|
|
5839
|
+
timeout: { type: "string" }
|
|
5840
|
+
}
|
|
5841
|
+
},
|
|
5767
5842
|
"datasources jobs list": {
|
|
5768
5843
|
usage: "Usage: remy-admin datasources jobs list [--source <slug>]",
|
|
5769
5844
|
flags: { source: { type: "string" } }
|
|
@@ -5867,6 +5942,65 @@ async function jobsStart(ctx, a) {
|
|
|
5867
5942
|
...job.autoApprove ? {} : { note: "Review the plan, then `datasources jobs approve <id>`." }
|
|
5868
5943
|
});
|
|
5869
5944
|
}
|
|
5945
|
+
function readDocuments(file) {
|
|
5946
|
+
const abs = path5.isAbsolute(file) ? file : path5.join(WORKSPACE_DIR, file);
|
|
5947
|
+
let content;
|
|
5948
|
+
try {
|
|
5949
|
+
content = fs5.readFileSync(abs, "utf8");
|
|
5950
|
+
} catch (err) {
|
|
5951
|
+
fatal(`Could not read "${file}": ${err.message}`);
|
|
5952
|
+
}
|
|
5953
|
+
const documents2 = [];
|
|
5954
|
+
content.split("\n").forEach((line, i) => {
|
|
5955
|
+
const trimmed = line.trim();
|
|
5956
|
+
if (!trimmed) {
|
|
5957
|
+
return;
|
|
5958
|
+
}
|
|
5959
|
+
let parsed;
|
|
5960
|
+
try {
|
|
5961
|
+
parsed = JSON.parse(trimmed);
|
|
5962
|
+
} catch {
|
|
5963
|
+
fatal(`${file}:${i + 1} is not valid JSON.`);
|
|
5964
|
+
}
|
|
5965
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
5966
|
+
fatal(
|
|
5967
|
+
`${file}:${i + 1} needs { "externalId": "...", "title": "...", "markdown": "..." }.`
|
|
5968
|
+
);
|
|
5969
|
+
}
|
|
5970
|
+
documents2.push(parsed);
|
|
5971
|
+
});
|
|
5972
|
+
if (documents2.length === 0) {
|
|
5973
|
+
fatal(`"${file}" has no documents.`);
|
|
5974
|
+
}
|
|
5975
|
+
return documents2;
|
|
5976
|
+
}
|
|
5977
|
+
async function addMany2(ctx, a) {
|
|
5978
|
+
const slug = sourceOf(a);
|
|
5979
|
+
const documents2 = readDocuments(a.required("file"));
|
|
5980
|
+
const budget = a.num("budget");
|
|
5981
|
+
const { documentCount, job } = await addMany(ctx, {
|
|
5982
|
+
slug,
|
|
5983
|
+
documents: documents2,
|
|
5984
|
+
...budget !== void 0 ? { budgetDollars: budget } : {},
|
|
5985
|
+
onProgress: (stored) => progress(
|
|
5986
|
+
`stored ${stored.toLocaleString()}/${documents2.length.toLocaleString()}\u2026`
|
|
5987
|
+
)
|
|
5988
|
+
});
|
|
5989
|
+
if (!job || !a.bool("wait")) {
|
|
5990
|
+
printJson({
|
|
5991
|
+
dataSource: slug,
|
|
5992
|
+
documentCount,
|
|
5993
|
+
job,
|
|
5994
|
+
note: "Planning, then running once the plan passes the gates. Follow it with `datasources jobs status <id> --wait`."
|
|
5995
|
+
});
|
|
5996
|
+
return;
|
|
5997
|
+
}
|
|
5998
|
+
await waitAndReport(ctx, a, job.id, "job", {
|
|
5999
|
+
dataSource: slug,
|
|
6000
|
+
action: "add-many",
|
|
6001
|
+
documentCount
|
|
6002
|
+
});
|
|
6003
|
+
}
|
|
5870
6004
|
async function jobsList(ctx, a) {
|
|
5871
6005
|
const slug = a.str("source");
|
|
5872
6006
|
printJson(await list11(ctx, slug ? { slug } : void 0));
|
|
@@ -5927,6 +6061,7 @@ async function jobsCancel(ctx, a) {
|
|
|
5927
6061
|
}
|
|
5928
6062
|
var dataSourceJobsHandlers = {
|
|
5929
6063
|
"datasources jobs start": jobsStart,
|
|
6064
|
+
"datasources add-many": addMany2,
|
|
5930
6065
|
"datasources jobs list": jobsList,
|
|
5931
6066
|
"datasources jobs status": jobsStatus,
|
|
5932
6067
|
"datasources jobs approve": jobsApprove,
|
|
@@ -5994,6 +6129,27 @@ Bulk ingestion (jobs):
|
|
|
5994
6129
|
a candidate version exists, and moves, re-vectorizes and deletes refuse while
|
|
5995
6130
|
a job is in flight (data_source_busy). Interactive \`add\` keeps working.
|
|
5996
6131
|
|
|
6132
|
+
Adding many documents (add-many):
|
|
6133
|
+
remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>
|
|
6134
|
+
|
|
6135
|
+
For records rendered to text, in any number: JSONL, one document per line,
|
|
6136
|
+
{"externalId": "claim-123", "title": "\u2026", "markdown": "\u2026", "metadata": {\u2026},
|
|
6137
|
+
"entities": ["\u2026"], "replaces": ["\u2026"]}. externalId, title and markdown are
|
|
6138
|
+
required. The file is stored a thousand documents at a time, then one job
|
|
6139
|
+
registers and builds them: it plans, then runs on its own once the plan
|
|
6140
|
+
passes the gates (the two \`jobs start\` has), within --budget. Follow it
|
|
6141
|
+
with \`jobs status <id>\`, or pass --wait.
|
|
6142
|
+
|
|
6143
|
+
externalId is a document's identity. One the source already holds is
|
|
6144
|
+
superseded when the job finishes; unchanged markdown builds nothing; new
|
|
6145
|
+
metadata or entities on unchanged markdown update in place. A document that
|
|
6146
|
+
fails validation is left out and counted in the job's quarantine
|
|
6147
|
+
(\`jobs quarantine <id>\`), named by its place in the file from 0; the
|
|
6148
|
+
rest go in. Nothing is removed for being left out of a later file.
|
|
6149
|
+
|
|
6150
|
+
Not on a source with a mapper (data_source_mapped): there the mapper makes
|
|
6151
|
+
the documents. One job per source at a time (data_source_busy).
|
|
6152
|
+
|
|
5997
6153
|
--wait exit codes: 0 done \xB7 1 the job failed, paused, was cancelled, or sits
|
|
5998
6154
|
in planned because a gate refused it (approve once the cause is fixed) \xB7
|
|
5999
6155
|
2 still running when --timeout elapsed (default 3600s).
|
|
@@ -7317,6 +7473,7 @@ Subcommands:
|
|
|
7317
7473
|
create Create an empty source, optionally on dedicated capacity
|
|
7318
7474
|
config Show or change how a corpus is processed and searched
|
|
7319
7475
|
move Move a corpus between shared and dedicated capacity, data intact
|
|
7476
|
+
add-many Add a file of documents (JSONL records rendered to text), built as one job
|
|
7320
7477
|
jobs Bulk-load a corpus from a file store or a manifest, with a plan first
|
|
7321
7478
|
connect Follow an S3 bucket the customer owns (credentials = app secret names)
|
|
7322
7479
|
sync Bring a connected source up to date with its bucket
|
|
@@ -7349,6 +7506,7 @@ Usage:
|
|
|
7349
7506
|
remy-admin datasources create --source <slug> [--name <name>] [--placement <resource-id|shared>] [rebuild settings...]
|
|
7350
7507
|
remy-admin datasources config [--source <slug>] [--placement <resource-id|shared>] [settings...]
|
|
7351
7508
|
remy-admin datasources move [--source <slug>] --to <resource-id|shared> [--chains <n>] [--restart] [--wait] [--timeout <sec>]
|
|
7509
|
+
remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>
|
|
7352
7510
|
remy-admin datasources jobs start|list|status|approve|pause|resume|retry|cancel \u2026 (see below)
|
|
7353
7511
|
remy-admin datasources connect|sync|connector|disconnect \u2026 (see below)
|
|
7354
7512
|
remy-admin datasources hydrate|reindex|sample \u2026 (see below)
|
|
@@ -7486,9 +7644,12 @@ Notes:
|
|
|
7486
7644
|
Exits ${EXIT.buildFailed} if a document failed, ${EXIT.timeout} on timeout.
|
|
7487
7645
|
Re-adding an unchanged file is free: no upload, no re-embedding. Re-adding
|
|
7488
7646
|
with a different --metadata updates the tags in place, also free.
|
|
7489
|
-
add
|
|
7490
|
-
|
|
7491
|
-
(
|
|
7647
|
+
add is refused with data_source_full (422) once a source holds what its
|
|
7648
|
+
placement does: a shared-pool source at the per-source cap moves to
|
|
7649
|
+
dedicated capacity (\`move\`), a dedicated one is resized. The message
|
|
7650
|
+
says which.
|
|
7651
|
+
add is for a file at a time. For records rendered to text in any number,
|
|
7652
|
+
use add-many; for files in a store or behind URLs, a job.
|
|
7492
7653
|
--metadata tags documents for search-time filtering (scalars, \u226416 keys).
|
|
7493
7654
|
--entities names what a document is about, as your own ids (\u2264256, no spaces
|
|
7494
7655
|
or commas); an empty --entities clears them. Also free on a re-add.
|
|
@@ -7528,6 +7689,9 @@ Examples:
|
|
|
7528
7689
|
remy-admin datasources jobs start --source archive --store raw --prefix 2024/ --wait
|
|
7529
7690
|
remy-admin datasources jobs approve <id> --wait
|
|
7530
7691
|
|
|
7692
|
+
# Add records rendered to text, in any number: one job, priced first
|
|
7693
|
+
remy-admin datasources add-many --source claims --wait claims.jsonl
|
|
7694
|
+
|
|
7531
7695
|
# Follow a customer's bucket: secrets by name, first sync stops for approval
|
|
7532
7696
|
remy-admin secrets set ARCHIVE_S3_KEY --prod <value> # never on a command line you share
|
|
7533
7697
|
remy-admin datasources connect --source archive --bucket acme-docs --region us-east-1 --prefix contracts/ --access-key-secret ARCHIVE_S3_KEY --secret-key-secret ARCHIVE_S3_SECRET --budget-per-sync 5
|
|
@@ -54,6 +54,10 @@ export type DataSourceJobSource = {
|
|
|
54
54
|
*/
|
|
55
55
|
type: 'object';
|
|
56
56
|
ref: string;
|
|
57
|
+
} | {
|
|
58
|
+
/** The documents a `datasources add-many` (or an app's `addMany()`) gave, and how many. */
|
|
59
|
+
type: 'documents';
|
|
60
|
+
documents: number;
|
|
57
61
|
};
|
|
58
62
|
/**
|
|
59
63
|
* planning → planned → running ⇄ paused → done | failed | cancelled.
|
|
@@ -272,6 +276,25 @@ export interface DataSourceJobGetResult {
|
|
|
272
276
|
export interface DataSourceJobResult {
|
|
273
277
|
job: DataSourceJob;
|
|
274
278
|
}
|
|
279
|
+
/** POST /datasources/add-many/open */
|
|
280
|
+
export interface DataSourceAddManyOpenResult {
|
|
281
|
+
/** Names the upload on `add-many/part` and `add-many/start`. */
|
|
282
|
+
uploadId: string;
|
|
283
|
+
}
|
|
284
|
+
/** POST /datasources/add-many/part */
|
|
285
|
+
export interface DataSourceAddManyPartResult {
|
|
286
|
+
/** The part's number, which `add-many/start` lists it by. */
|
|
287
|
+
part: number;
|
|
288
|
+
uploadUrl: string;
|
|
289
|
+
uploadFields: Record<string, string>;
|
|
290
|
+
}
|
|
291
|
+
/** What `addMany` answers with. */
|
|
292
|
+
export interface DataSourceAddManyResult {
|
|
293
|
+
/** How many documents were given. */
|
|
294
|
+
documentCount: number;
|
|
295
|
+
/** The job registering and building them; null only when none were given. */
|
|
296
|
+
job: DataSourceJob | null;
|
|
297
|
+
}
|
|
275
298
|
/** POST /datasources/jobs/manifest-token */
|
|
276
299
|
export interface DataSourceJobManifestTokenResult {
|
|
277
300
|
key: string;
|
|
@@ -352,11 +352,6 @@ export interface DataSourcesListEntry {
|
|
|
352
352
|
name: string | null;
|
|
353
353
|
/** Total document count (active pipeline). */
|
|
354
354
|
documentCount: number;
|
|
355
|
-
/**
|
|
356
|
-
* The most documents the source may hold, the app's limit; null when
|
|
357
|
-
* unlimited. An add past it is refused with `document_limit_reached`.
|
|
358
|
-
*/
|
|
359
|
-
documentLimit: number | null;
|
|
360
355
|
counts: DataSourcesBuildProgress;
|
|
361
356
|
/** Source-document bytes (does not include vector storage). */
|
|
362
357
|
storageBytes: number;
|