@madewithremy/admin 0.1.67 → 0.1.69

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,7 @@
7
7
  * commands/dataSourceJobs.ts owns flags, progress printing and exit codes.
8
8
  */
9
9
  import type { AdminContext } from '../ctx.js';
10
- import type { DataSourceJob, DataSourceJobGetResult, DataSourceJobResult, DataSourceJobsListResult, DataSourceJobSource } from '../types/dataSourceJobs.js';
10
+ import type { DataSourceAddManyResult, DataSourceJob, DataSourceJobGetResult, DataSourceJobResult, DataSourceJobsListResult, DataSourceJobSource } from '../types/dataSourceJobs.js';
11
11
  /** @internal Consumed by the CLI skin. */
12
12
  export declare const DEFAULT_WAIT_TIMEOUT_MS: number;
13
13
  export interface StartParams {
@@ -54,6 +54,57 @@ export interface UploadManifestParams {
54
54
  export declare function uploadManifest(ctx: AdminContext, params: UploadManifestParams): Promise<{
55
55
  key: string;
56
56
  }>;
57
+ /** One document of `addMany`: what a mapper's `documents()` returns. */
58
+ export interface AddManyDocument {
59
+ /**
60
+ * The document's identity. Giving one the source already holds supersedes
61
+ * that document.
62
+ */
63
+ externalId: string;
64
+ title: string;
65
+ /** The document's text. Chunked and embedded as markdown. */
66
+ markdown: string;
67
+ /** Scalars only, up to 16 keys; filterable at search time. */
68
+ metadata?: Record<string, string | number | boolean>;
69
+ /** What the document is about, as your own ids. */
70
+ entities?: string[];
71
+ /** External ids this document replaces. */
72
+ replaces?: string | string[];
73
+ }
74
+ export interface AddManyParams {
75
+ /** Data source slug. Created on first use, like `add`. */
76
+ slug: string;
77
+ documents: Iterable<AddManyDocument> | AsyncIterable<AddManyDocument>;
78
+ /** Pause when estimated spend reaches this; default 1.5× the projection. */
79
+ budgetDollars?: number;
80
+ /** Called after each part is stored, with how many documents are stored so far. */
81
+ onProgress?: (stored: number) => void;
82
+ }
83
+ /**
84
+ * Add many documents at once, built as one job: records rendered to text
85
+ * (a row of the app's own table as a markdown card), in any number. They are
86
+ * stored a thousand at a time, then one job registers and builds them. The
87
+ * job plans, and starts on its own once its plan passes the gates, within
88
+ * `budgetDollars`. Returns once the job has started; follow it with `get` or
89
+ * `waitForJob`.
90
+ *
91
+ * An external id the source already holds is superseded when the job
92
+ * finishes; unchanged markdown builds nothing, and new metadata or entities
93
+ * on unchanged markdown update in place. A document that fails validation is
94
+ * left out and counted in the job's quarantine; the rest go in. Nothing is
95
+ * removed for being left out of a later call.
96
+ *
97
+ * @throws AdminApiError `data_source_mapped` (422) — the source has a mapper,
98
+ * which is what makes its documents; `data_source_busy` (422) — a move, a
99
+ * candidate version or another job is in flight; `invalid_parts` (400);
100
+ * `add_many_not_found` (404) — the upload took longer than two hours.
101
+ * @example
102
+ * const { job } = await admin.dataSourceJobs.addMany({
103
+ * slug: 'claims',
104
+ * documents: [{ externalId: 'c-1', title: 'Claim 1', markdown: '…' }],
105
+ * });
106
+ */
107
+ export declare function addMany(ctx: AdminContext, params: AddManyParams): Promise<DataSourceAddManyResult>;
57
108
  export interface ListParams {
58
109
  /** Only this source's jobs; omit for every job on the app. */
59
110
  slug?: string;
@@ -97,8 +97,9 @@ export interface AddDocumentResult {
97
97
  *
98
98
  * @throws AdminApiError `invalid_content_hash` (400) — malformed hash
99
99
  * (internal; `addDocument` computes this from `content`).
100
- * @throws AdminApiError `document_limit_reached` (422) — the source holds the
101
- * app's limit of documents (`documentLimit` in `list`); support raises it.
100
+ * @throws AdminApiError `data_source_full` (422) — the source holds what its
101
+ * placement does: the shared pool's chunks per source, or its dedicated
102
+ * capacity's room. The message says which, and the way out.
102
103
  * @example
103
104
  * const result = await admin.dataSources.addDocument({
104
105
  * slug: 'policies',
@@ -217,8 +218,7 @@ export declare function documents(ctx: AdminContext, params: DocumentsParams): P
217
218
  */
218
219
  export declare function allDocuments(ctx: AdminContext, params: Omit<DocumentsParams, 'cursor' | 'limit'>): Promise<DataSourcesDocumentStatus[]>;
219
220
  /**
220
- * List all data sources with document counts, the most documents each may
221
- * hold (`documentLimit`, null when unlimited), build progress, and active
221
+ * List all data sources with document counts, build progress, and active
222
222
  * version metadata.
223
223
  *
224
224
  * @example
package/dist/ops/db.d.ts CHANGED
@@ -19,6 +19,10 @@ import type { DbHistoryPurgeResult, DbHistoryResult, DbQueryResult } from '../ty
19
19
  * too large) throws with `errorString: 'sql_error'`, and the body's
20
20
  * `errorMessage` is the database's own message.
21
21
  *
22
+ * A statement that changes live is recorded in the workspace's audit log:
23
+ * who ran it and when, which database and how many rows changed, never the
24
+ * SQL.
25
+ *
22
26
  * @param sql The SQL statement to execute.
23
27
  * @throws AdminApiError `no_live_release` (404) — the app has not been
24
28
  * deployed; `no_database` (404) — the live release has no database;
@@ -70,9 +74,10 @@ export declare function historyQuery(ctx: AdminContext, releaseId: string, sql:
70
74
  /**
71
75
  * Run one SQL statement against live with a past release's database attached
72
76
  * as the schema `history`: `history.<table>` is the past table, and an
73
- * unqualified name is live's. Writes to live are real, as with `query`, and
74
- * run in one transaction; the attached copy is private to the call, so
75
- * nothing written into it lasts.
77
+ * unqualified name is live's. Writes to live are real, as with `query`, run
78
+ * in one transaction, and are recorded in the workspace's audit log as
79
+ * `query`'s are; the attached copy is private to the call, so nothing
80
+ * written into it lasts.
76
81
  *
77
82
  * A past database larger than 1 GB can't be attached: read it with
78
83
  * `historyQuery` instead, and write what's needed to live with `query`.
package/dist/prod.js CHANGED
@@ -2457,6 +2457,10 @@ Notes:
2457
2457
  SQL don't parse flags, so a statement may safely begin with a '--'
2458
2458
  comment. Always quote the SQL.
2459
2459
  - The history commands use the database named as live's default.
2460
+ - A 'db query' or 'db history attach' that changes live is recorded in the
2461
+ workspace's audit log, as a purge is: who ran it and when, which database
2462
+ and how many rows changed, never the SQL. Workspace admins read the log
2463
+ in Workspace Settings.
2460
2464
  - The first history command on a copy fetches it, so it takes longer for a
2461
2465
  large database; the ones after it are quick.
2462
2466
  - Errors: release_not_found (no release of this app that was ever live has
@@ -4419,6 +4423,7 @@ function link(ctx, params) {
4419
4423
  }
4420
4424
 
4421
4425
  // src/ops/dataSourceJobs.ts
4426
+ import { gzipSync } from "node:zlib";
4422
4427
  var DEFAULT_WAIT_TIMEOUT_MS2 = 60 * 60 * 1e3;
4423
4428
  var POLL_MS3 = 5e3;
4424
4429
  function base2(appId) {
@@ -4446,6 +4451,66 @@ async function uploadManifest(ctx, params) {
4446
4451
  await uploadDirect(token.upload, params.content, "manifest.jsonl");
4447
4452
  return { key: token.key };
4448
4453
  }
4454
+ var PART_DOCUMENTS = 1e3;
4455
+ var PART_BYTES = 8 * 1024 * 1024;
4456
+ async function addMany(ctx, params) {
4457
+ const addManyBase = `/_internal/v2/apps/${ctx.appId}/datasources/add-many`;
4458
+ const { uploadId } = await call(
4459
+ ctx,
4460
+ "POST",
4461
+ `${addManyBase}/open`,
4462
+ { slug: params.slug }
4463
+ );
4464
+ const parts = [];
4465
+ let lines = [];
4466
+ let bytes = 0;
4467
+ let documentCount = 0;
4468
+ const storeLines = async () => {
4469
+ if (lines.length === 0) {
4470
+ return;
4471
+ }
4472
+ const grant = await call(
4473
+ ctx,
4474
+ "POST",
4475
+ `${addManyBase}/part`,
4476
+ { slug: params.slug, uploadId }
4477
+ );
4478
+ await uploadDirect(
4479
+ grant,
4480
+ gzipSync(Buffer.from(lines.join("\n") + "\n")),
4481
+ `${grant.part}.ndjson.gz`
4482
+ );
4483
+ parts.push({ part: grant.part, count: lines.length });
4484
+ lines = [];
4485
+ bytes = 0;
4486
+ params.onProgress?.(documentCount);
4487
+ };
4488
+ for await (const d of params.documents) {
4489
+ const line = JSON.stringify({ o: 0, i: documentCount, d });
4490
+ lines.push(line);
4491
+ bytes += line.length;
4492
+ documentCount += 1;
4493
+ if (lines.length >= PART_DOCUMENTS || bytes >= PART_BYTES) {
4494
+ await storeLines();
4495
+ }
4496
+ }
4497
+ await storeLines();
4498
+ if (parts.length === 0) {
4499
+ return { documentCount: 0, job: null };
4500
+ }
4501
+ const { job } = await call(
4502
+ ctx,
4503
+ "POST",
4504
+ `${addManyBase}/start`,
4505
+ {
4506
+ slug: params.slug,
4507
+ uploadId,
4508
+ parts,
4509
+ ...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {}
4510
+ }
4511
+ );
4512
+ return { documentCount, job };
4513
+ }
4449
4514
  function list11(ctx, params) {
4450
4515
  return call(
4451
4516
  ctx,
@@ -5764,6 +5829,16 @@ var dataSourceJobsSpecs = {
5764
5829
  message: "Give the job something to read: --store <name> [--prefix <p>], or --manifest <file.jsonl>."
5765
5830
  }
5766
5831
  },
5832
+ "datasources add-many": {
5833
+ usage: "Usage: remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>",
5834
+ positionals: [{ name: "file", required: true }],
5835
+ flags: {
5836
+ source: { type: "string" },
5837
+ budget: { type: "number", min: 0.01 },
5838
+ wait: { type: "boolean" },
5839
+ timeout: { type: "string" }
5840
+ }
5841
+ },
5767
5842
  "datasources jobs list": {
5768
5843
  usage: "Usage: remy-admin datasources jobs list [--source <slug>]",
5769
5844
  flags: { source: { type: "string" } }
@@ -5867,6 +5942,65 @@ async function jobsStart(ctx, a) {
5867
5942
  ...job.autoApprove ? {} : { note: "Review the plan, then `datasources jobs approve <id>`." }
5868
5943
  });
5869
5944
  }
5945
+ function readDocuments(file) {
5946
+ const abs = path5.isAbsolute(file) ? file : path5.join(WORKSPACE_DIR, file);
5947
+ let content;
5948
+ try {
5949
+ content = fs5.readFileSync(abs, "utf8");
5950
+ } catch (err) {
5951
+ fatal(`Could not read "${file}": ${err.message}`);
5952
+ }
5953
+ const documents2 = [];
5954
+ content.split("\n").forEach((line, i) => {
5955
+ const trimmed = line.trim();
5956
+ if (!trimmed) {
5957
+ return;
5958
+ }
5959
+ let parsed;
5960
+ try {
5961
+ parsed = JSON.parse(trimmed);
5962
+ } catch {
5963
+ fatal(`${file}:${i + 1} is not valid JSON.`);
5964
+ }
5965
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
5966
+ fatal(
5967
+ `${file}:${i + 1} needs { "externalId": "...", "title": "...", "markdown": "..." }.`
5968
+ );
5969
+ }
5970
+ documents2.push(parsed);
5971
+ });
5972
+ if (documents2.length === 0) {
5973
+ fatal(`"${file}" has no documents.`);
5974
+ }
5975
+ return documents2;
5976
+ }
5977
+ async function addMany2(ctx, a) {
5978
+ const slug = sourceOf(a);
5979
+ const documents2 = readDocuments(a.required("file"));
5980
+ const budget = a.num("budget");
5981
+ const { documentCount, job } = await addMany(ctx, {
5982
+ slug,
5983
+ documents: documents2,
5984
+ ...budget !== void 0 ? { budgetDollars: budget } : {},
5985
+ onProgress: (stored) => progress(
5986
+ `stored ${stored.toLocaleString()}/${documents2.length.toLocaleString()}\u2026`
5987
+ )
5988
+ });
5989
+ if (!job || !a.bool("wait")) {
5990
+ printJson({
5991
+ dataSource: slug,
5992
+ documentCount,
5993
+ job,
5994
+ note: "Planning, then running once the plan passes the gates. Follow it with `datasources jobs status <id> --wait`."
5995
+ });
5996
+ return;
5997
+ }
5998
+ await waitAndReport(ctx, a, job.id, "job", {
5999
+ dataSource: slug,
6000
+ action: "add-many",
6001
+ documentCount
6002
+ });
6003
+ }
5870
6004
  async function jobsList(ctx, a) {
5871
6005
  const slug = a.str("source");
5872
6006
  printJson(await list11(ctx, slug ? { slug } : void 0));
@@ -5927,6 +6061,7 @@ async function jobsCancel(ctx, a) {
5927
6061
  }
5928
6062
  var dataSourceJobsHandlers = {
5929
6063
  "datasources jobs start": jobsStart,
6064
+ "datasources add-many": addMany2,
5930
6065
  "datasources jobs list": jobsList,
5931
6066
  "datasources jobs status": jobsStatus,
5932
6067
  "datasources jobs approve": jobsApprove,
@@ -5994,6 +6129,27 @@ Bulk ingestion (jobs):
5994
6129
  a candidate version exists, and moves, re-vectorizes and deletes refuse while
5995
6130
  a job is in flight (data_source_busy). Interactive \`add\` keeps working.
5996
6131
 
6132
+ Adding many documents (add-many):
6133
+ remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>
6134
+
6135
+ For records rendered to text, in any number: JSONL, one document per line,
6136
+ {"externalId": "claim-123", "title": "\u2026", "markdown": "\u2026", "metadata": {\u2026},
6137
+ "entities": ["\u2026"], "replaces": ["\u2026"]}. externalId, title and markdown are
6138
+ required. The file is stored a thousand documents at a time, then one job
6139
+ registers and builds them: it plans, then runs on its own once the plan
6140
+ passes the gates (the two \`jobs start\` has), within --budget. Follow it
6141
+ with \`jobs status <id>\`, or pass --wait.
6142
+
6143
+ externalId is a document's identity. One the source already holds is
6144
+ superseded when the job finishes; unchanged markdown builds nothing; new
6145
+ metadata or entities on unchanged markdown update in place. A document that
6146
+ fails validation is left out and counted in the job's quarantine
6147
+ (\`jobs quarantine <id>\`), named by its place in the file from 0; the
6148
+ rest go in. Nothing is removed for being left out of a later file.
6149
+
6150
+ Not on a source with a mapper (data_source_mapped): there the mapper makes
6151
+ the documents. One job per source at a time (data_source_busy).
6152
+
5997
6153
  --wait exit codes: 0 done \xB7 1 the job failed, paused, was cancelled, or sits
5998
6154
  in planned because a gate refused it (approve once the cause is fixed) \xB7
5999
6155
  2 still running when --timeout elapsed (default 3600s).
@@ -7317,6 +7473,7 @@ Subcommands:
7317
7473
  create Create an empty source, optionally on dedicated capacity
7318
7474
  config Show or change how a corpus is processed and searched
7319
7475
  move Move a corpus between shared and dedicated capacity, data intact
7476
+ add-many Add a file of documents (JSONL records rendered to text), built as one job
7320
7477
  jobs Bulk-load a corpus from a file store or a manifest, with a plan first
7321
7478
  connect Follow an S3 bucket the customer owns (credentials = app secret names)
7322
7479
  sync Bring a connected source up to date with its bucket
@@ -7349,6 +7506,7 @@ Usage:
7349
7506
  remy-admin datasources create --source <slug> [--name <name>] [--placement <resource-id|shared>] [rebuild settings...]
7350
7507
  remy-admin datasources config [--source <slug>] [--placement <resource-id|shared>] [settings...]
7351
7508
  remy-admin datasources move [--source <slug>] --to <resource-id|shared> [--chains <n>] [--restart] [--wait] [--timeout <sec>]
7509
+ remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>
7352
7510
  remy-admin datasources jobs start|list|status|approve|pause|resume|retry|cancel \u2026 (see below)
7353
7511
  remy-admin datasources connect|sync|connector|disconnect \u2026 (see below)
7354
7512
  remy-admin datasources hydrate|reindex|sample \u2026 (see below)
@@ -7486,9 +7644,12 @@ Notes:
7486
7644
  Exits ${EXIT.buildFailed} if a document failed, ${EXIT.timeout} on timeout.
7487
7645
  Re-adding an unchanged file is free: no upload, no re-embedding. Re-adding
7488
7646
  with a different --metadata updates the tags in place, also free.
7489
- add stops at the app's documentLimit, the most documents a source may hold
7490
- (shown by \`list\`): past it, add is refused with document_limit_reached
7491
- (422), and support raises the app's limit. Bulk jobs aren't held to it.
7647
+ add is refused with data_source_full (422) once a source holds what its
7648
+ placement does: a shared-pool source at the per-source cap moves to
7649
+ dedicated capacity (\`move\`), a dedicated one is resized. The message
7650
+ says which.
7651
+ add is for a file at a time. For records rendered to text in any number,
7652
+ use add-many; for files in a store or behind URLs, a job.
7492
7653
  --metadata tags documents for search-time filtering (scalars, \u226416 keys).
7493
7654
  --entities names what a document is about, as your own ids (\u2264256, no spaces
7494
7655
  or commas); an empty --entities clears them. Also free on a re-add.
@@ -7528,6 +7689,9 @@ Examples:
7528
7689
  remy-admin datasources jobs start --source archive --store raw --prefix 2024/ --wait
7529
7690
  remy-admin datasources jobs approve <id> --wait
7530
7691
 
7692
+ # Add records rendered to text, in any number: one job, priced first
7693
+ remy-admin datasources add-many --source claims --wait claims.jsonl
7694
+
7531
7695
  # Follow a customer's bucket: secrets by name, first sync stops for approval
7532
7696
  remy-admin secrets set ARCHIVE_S3_KEY --prod <value> # never on a command line you share
7533
7697
  remy-admin datasources connect --source archive --bucket acme-docs --region us-east-1 --prefix contracts/ --access-key-secret ARCHIVE_S3_KEY --secret-key-secret ARCHIVE_S3_SECRET --budget-per-sync 5
@@ -54,6 +54,10 @@ export type DataSourceJobSource = {
54
54
  */
55
55
  type: 'object';
56
56
  ref: string;
57
+ } | {
58
+ /** The documents a `datasources add-many` (or an app's `addMany()`) gave, and how many. */
59
+ type: 'documents';
60
+ documents: number;
57
61
  };
58
62
  /**
59
63
  * planning → planned → running ⇄ paused → done | failed | cancelled.
@@ -272,6 +276,25 @@ export interface DataSourceJobGetResult {
272
276
  export interface DataSourceJobResult {
273
277
  job: DataSourceJob;
274
278
  }
279
+ /** POST /datasources/add-many/open */
280
+ export interface DataSourceAddManyOpenResult {
281
+ /** Names the upload on `add-many/part` and `add-many/start`. */
282
+ uploadId: string;
283
+ }
284
+ /** POST /datasources/add-many/part */
285
+ export interface DataSourceAddManyPartResult {
286
+ /** The part's number, which `add-many/start` lists it by. */
287
+ part: number;
288
+ uploadUrl: string;
289
+ uploadFields: Record<string, string>;
290
+ }
291
+ /** What `addMany` answers with. */
292
+ export interface DataSourceAddManyResult {
293
+ /** How many documents were given. */
294
+ documentCount: number;
295
+ /** The job registering and building them; null only when none were given. */
296
+ job: DataSourceJob | null;
297
+ }
275
298
  /** POST /datasources/jobs/manifest-token */
276
299
  export interface DataSourceJobManifestTokenResult {
277
300
  key: string;
@@ -352,11 +352,6 @@ export interface DataSourcesListEntry {
352
352
  name: string | null;
353
353
  /** Total document count (active pipeline). */
354
354
  documentCount: number;
355
- /**
356
- * The most documents the source may hold, the app's limit; null when
357
- * unlimited. An add past it is refused with `document_limit_reached`.
358
- */
359
- documentLimit: number | null;
360
355
  counts: DataSourcesBuildProgress;
361
356
  /** Source-document bytes (does not include vector storage). */
362
357
  storageBytes: number;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@madewithremy/admin",
3
- "version": "0.1.67",
3
+ "version": "0.1.69",
4
4
  "description": "Admin SDK + CLI for managing production Remy apps",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",