@madewithremy/admin 0.1.67 → 0.1.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,7 @@
7
7
  * commands/dataSourceJobs.ts owns flags, progress printing and exit codes.
8
8
  */
9
9
  import type { AdminContext } from '../ctx.js';
10
- import type { DataSourceJob, DataSourceJobGetResult, DataSourceJobResult, DataSourceJobsListResult, DataSourceJobSource } from '../types/dataSourceJobs.js';
10
+ import type { DataSourceAddManyResult, DataSourceJob, DataSourceJobGetResult, DataSourceJobResult, DataSourceJobsListResult, DataSourceJobSource } from '../types/dataSourceJobs.js';
11
11
  /** @internal Consumed by the CLI skin. */
12
12
  export declare const DEFAULT_WAIT_TIMEOUT_MS: number;
13
13
  export interface StartParams {
@@ -54,6 +54,57 @@ export interface UploadManifestParams {
54
54
  export declare function uploadManifest(ctx: AdminContext, params: UploadManifestParams): Promise<{
55
55
  key: string;
56
56
  }>;
57
+ /** One document of `addMany`: what a mapper's `documents()` returns. */
58
+ export interface AddManyDocument {
59
+ /**
60
+ * The document's identity. Giving one the source already holds supersedes
61
+ * that document.
62
+ */
63
+ externalId: string;
64
+ title: string;
65
+ /** The document's text. Chunked and embedded as markdown. */
66
+ markdown: string;
67
+ /** Scalars only, up to 16 keys; filterable at search time. */
68
+ metadata?: Record<string, string | number | boolean>;
69
+ /** What the document is about, as your own ids. */
70
+ entities?: string[];
71
+ /** External ids this document replaces. */
72
+ replaces?: string | string[];
73
+ }
74
+ export interface AddManyParams {
75
+ /** Data source slug. Created on first use, like `add`. */
76
+ slug: string;
77
+ documents: Iterable<AddManyDocument> | AsyncIterable<AddManyDocument>;
78
+ /** Pause when estimated spend reaches this; default 1.5× the projection. */
79
+ budgetDollars?: number;
80
+ /** Called after each part is stored, with how many documents are stored so far. */
81
+ onProgress?: (stored: number) => void;
82
+ }
83
+ /**
84
+ * Add many documents at once, built as one job: records rendered to text
85
+ * (a row of the app's own table as a markdown card), in any number. They are
86
+ * stored a thousand at a time, then one job registers and builds them. The
87
+ * job plans, and starts on its own once its plan passes the gates, within
88
+ * `budgetDollars`. Returns once the job has started; follow it with `get` or
89
+ * `waitForJob`.
90
+ *
91
+ * An external id the source already holds is superseded when the job
92
+ * finishes; unchanged markdown builds nothing, and new metadata or entities
93
+ * on unchanged markdown update in place. A document that fails validation is
94
+ * left out and counted in the job's quarantine; the rest go in. Nothing is
95
+ * removed for being left out of a later call.
96
+ *
97
+ * @throws AdminApiError `data_source_mapped` (422) — the source has a mapper,
98
+ * which is what makes its documents; `data_source_busy` (422) — a move, a
99
+ * candidate version or another job is in flight; `invalid_parts` (400);
100
+ * `add_many_not_found` (404) — the upload took longer than two hours.
101
+ * @example
102
+ * const { job } = await admin.dataSourceJobs.addMany({
103
+ * slug: 'claims',
104
+ * documents: [{ externalId: 'c-1', title: 'Claim 1', markdown: '…' }],
105
+ * });
106
+ */
107
+ export declare function addMany(ctx: AdminContext, params: AddManyParams): Promise<DataSourceAddManyResult>;
57
108
  export interface ListParams {
58
109
  /** Only this source's jobs; omit for every job on the app. */
59
110
  slug?: string;
@@ -97,8 +97,9 @@ export interface AddDocumentResult {
97
97
  *
98
98
  * @throws AdminApiError `invalid_content_hash` (400) — malformed hash
99
99
  * (internal; `addDocument` computes this from `content`).
100
- * @throws AdminApiError `document_limit_reached` (422) — the source holds the
101
- * app's limit of documents (`documentLimit` in `list`); support raises it.
100
+ * @throws AdminApiError `data_source_full` (422) — the source holds what its
101
+ * placement does: the shared pool's chunks per source, or its dedicated
102
+ * capacity's room. The message says which, and the way out.
102
103
  * @example
103
104
  * const result = await admin.dataSources.addDocument({
104
105
  * slug: 'policies',
@@ -217,8 +218,7 @@ export declare function documents(ctx: AdminContext, params: DocumentsParams): P
217
218
  */
218
219
  export declare function allDocuments(ctx: AdminContext, params: Omit<DocumentsParams, 'cursor' | 'limit'>): Promise<DataSourcesDocumentStatus[]>;
219
220
  /**
220
- * List all data sources with document counts, the most documents each may
221
- * hold (`documentLimit`, null when unlimited), build progress, and active
221
+ * List all data sources with document counts, build progress, and active
222
222
  * version metadata.
223
223
  *
224
224
  * @example
package/dist/prod.js CHANGED
@@ -4419,6 +4419,7 @@ function link(ctx, params) {
4419
4419
  }
4420
4420
 
4421
4421
  // src/ops/dataSourceJobs.ts
4422
+ import { gzipSync } from "node:zlib";
4422
4423
  var DEFAULT_WAIT_TIMEOUT_MS2 = 60 * 60 * 1e3;
4423
4424
  var POLL_MS3 = 5e3;
4424
4425
  function base2(appId) {
@@ -4446,6 +4447,66 @@ async function uploadManifest(ctx, params) {
4446
4447
  await uploadDirect(token.upload, params.content, "manifest.jsonl");
4447
4448
  return { key: token.key };
4448
4449
  }
4450
+ var PART_DOCUMENTS = 1e3;
4451
+ var PART_BYTES = 8 * 1024 * 1024;
4452
+ async function addMany(ctx, params) {
4453
+ const addManyBase = `/_internal/v2/apps/${ctx.appId}/datasources/add-many`;
4454
+ const { uploadId } = await call(
4455
+ ctx,
4456
+ "POST",
4457
+ `${addManyBase}/open`,
4458
+ { slug: params.slug }
4459
+ );
4460
+ const parts = [];
4461
+ let lines = [];
4462
+ let bytes = 0;
4463
+ let documentCount = 0;
4464
+ const storeLines = async () => {
4465
+ if (lines.length === 0) {
4466
+ return;
4467
+ }
4468
+ const grant = await call(
4469
+ ctx,
4470
+ "POST",
4471
+ `${addManyBase}/part`,
4472
+ { slug: params.slug, uploadId }
4473
+ );
4474
+ await uploadDirect(
4475
+ grant,
4476
+ gzipSync(Buffer.from(lines.join("\n") + "\n")),
4477
+ `${grant.part}.ndjson.gz`
4478
+ );
4479
+ parts.push({ part: grant.part, count: lines.length });
4480
+ lines = [];
4481
+ bytes = 0;
4482
+ params.onProgress?.(documentCount);
4483
+ };
4484
+ for await (const d of params.documents) {
4485
+ const line = JSON.stringify({ o: 0, i: documentCount, d });
4486
+ lines.push(line);
4487
+ bytes += line.length;
4488
+ documentCount += 1;
4489
+ if (lines.length >= PART_DOCUMENTS || bytes >= PART_BYTES) {
4490
+ await storeLines();
4491
+ }
4492
+ }
4493
+ await storeLines();
4494
+ if (parts.length === 0) {
4495
+ return { documentCount: 0, job: null };
4496
+ }
4497
+ const { job } = await call(
4498
+ ctx,
4499
+ "POST",
4500
+ `${addManyBase}/start`,
4501
+ {
4502
+ slug: params.slug,
4503
+ uploadId,
4504
+ parts,
4505
+ ...params.budgetDollars !== void 0 ? { budgetDollars: params.budgetDollars } : {}
4506
+ }
4507
+ );
4508
+ return { documentCount, job };
4509
+ }
4449
4510
  function list11(ctx, params) {
4450
4511
  return call(
4451
4512
  ctx,
@@ -5764,6 +5825,16 @@ var dataSourceJobsSpecs = {
5764
5825
  message: "Give the job something to read: --store <name> [--prefix <p>], or --manifest <file.jsonl>."
5765
5826
  }
5766
5827
  },
5828
+ "datasources add-many": {
5829
+ usage: "Usage: remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>",
5830
+ positionals: [{ name: "file", required: true }],
5831
+ flags: {
5832
+ source: { type: "string" },
5833
+ budget: { type: "number", min: 0.01 },
5834
+ wait: { type: "boolean" },
5835
+ timeout: { type: "string" }
5836
+ }
5837
+ },
5767
5838
  "datasources jobs list": {
5768
5839
  usage: "Usage: remy-admin datasources jobs list [--source <slug>]",
5769
5840
  flags: { source: { type: "string" } }
@@ -5867,6 +5938,65 @@ async function jobsStart(ctx, a) {
5867
5938
  ...job.autoApprove ? {} : { note: "Review the plan, then `datasources jobs approve <id>`." }
5868
5939
  });
5869
5940
  }
5941
+ function readDocuments(file) {
5942
+ const abs = path5.isAbsolute(file) ? file : path5.join(WORKSPACE_DIR, file);
5943
+ let content;
5944
+ try {
5945
+ content = fs5.readFileSync(abs, "utf8");
5946
+ } catch (err) {
5947
+ fatal(`Could not read "${file}": ${err.message}`);
5948
+ }
5949
+ const documents2 = [];
5950
+ content.split("\n").forEach((line, i) => {
5951
+ const trimmed = line.trim();
5952
+ if (!trimmed) {
5953
+ return;
5954
+ }
5955
+ let parsed;
5956
+ try {
5957
+ parsed = JSON.parse(trimmed);
5958
+ } catch {
5959
+ fatal(`${file}:${i + 1} is not valid JSON.`);
5960
+ }
5961
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
5962
+ fatal(
5963
+ `${file}:${i + 1} needs { "externalId": "...", "title": "...", "markdown": "..." }.`
5964
+ );
5965
+ }
5966
+ documents2.push(parsed);
5967
+ });
5968
+ if (documents2.length === 0) {
5969
+ fatal(`"${file}" has no documents.`);
5970
+ }
5971
+ return documents2;
5972
+ }
5973
+ async function addMany2(ctx, a) {
5974
+ const slug = sourceOf(a);
5975
+ const documents2 = readDocuments(a.required("file"));
5976
+ const budget = a.num("budget");
5977
+ const { documentCount, job } = await addMany(ctx, {
5978
+ slug,
5979
+ documents: documents2,
5980
+ ...budget !== void 0 ? { budgetDollars: budget } : {},
5981
+ onProgress: (stored) => progress(
5982
+ `stored ${stored.toLocaleString()}/${documents2.length.toLocaleString()}\u2026`
5983
+ )
5984
+ });
5985
+ if (!job || !a.bool("wait")) {
5986
+ printJson({
5987
+ dataSource: slug,
5988
+ documentCount,
5989
+ job,
5990
+ note: "Planning, then running once the plan passes the gates. Follow it with `datasources jobs status <id> --wait`."
5991
+ });
5992
+ return;
5993
+ }
5994
+ await waitAndReport(ctx, a, job.id, "job", {
5995
+ dataSource: slug,
5996
+ action: "add-many",
5997
+ documentCount
5998
+ });
5999
+ }
5870
6000
  async function jobsList(ctx, a) {
5871
6001
  const slug = a.str("source");
5872
6002
  printJson(await list11(ctx, slug ? { slug } : void 0));
@@ -5927,6 +6057,7 @@ async function jobsCancel(ctx, a) {
5927
6057
  }
5928
6058
  var dataSourceJobsHandlers = {
5929
6059
  "datasources jobs start": jobsStart,
6060
+ "datasources add-many": addMany2,
5930
6061
  "datasources jobs list": jobsList,
5931
6062
  "datasources jobs status": jobsStatus,
5932
6063
  "datasources jobs approve": jobsApprove,
@@ -5994,6 +6125,27 @@ Bulk ingestion (jobs):
5994
6125
  a candidate version exists, and moves, re-vectorizes and deletes refuse while
5995
6126
  a job is in flight (data_source_busy). Interactive \`add\` keeps working.
5996
6127
 
6128
+ Adding many documents (add-many):
6129
+ remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>
6130
+
6131
+ For records rendered to text, in any number: JSONL, one document per line,
6132
+ {"externalId": "claim-123", "title": "\u2026", "markdown": "\u2026", "metadata": {\u2026},
6133
+ "entities": ["\u2026"], "replaces": ["\u2026"]}. externalId, title and markdown are
6134
+ required. The file is stored a thousand documents at a time, then one job
6135
+ registers and builds them: it plans, then runs on its own once the plan
6136
+ passes the gates (the two \`jobs start\` has), within --budget. Follow it
6137
+ with \`jobs status <id>\`, or pass --wait.
6138
+
6139
+ externalId is a document's identity. One the source already holds is
6140
+ superseded when the job finishes; unchanged markdown builds nothing; new
6141
+ metadata or entities on unchanged markdown update in place. A document that
6142
+ fails validation is left out and counted in the job's quarantine
6143
+ (\`jobs quarantine <id>\`), named by its place in the file from 0; the
6144
+ rest go in. Nothing is removed for being left out of a later file.
6145
+
6146
+ Not on a source with a mapper (data_source_mapped): there the mapper makes
6147
+ the documents. One job per source at a time (data_source_busy).
6148
+
5997
6149
  --wait exit codes: 0 done \xB7 1 the job failed, paused, was cancelled, or sits
5998
6150
  in planned because a gate refused it (approve once the cause is fixed) \xB7
5999
6151
  2 still running when --timeout elapsed (default 3600s).
@@ -7317,6 +7469,7 @@ Subcommands:
7317
7469
  create Create an empty source, optionally on dedicated capacity
7318
7470
  config Show or change how a corpus is processed and searched
7319
7471
  move Move a corpus between shared and dedicated capacity, data intact
7472
+ add-many Add a file of documents (JSONL records rendered to text), built as one job
7320
7473
  jobs Bulk-load a corpus from a file store or a manifest, with a plan first
7321
7474
  connect Follow an S3 bucket the customer owns (credentials = app secret names)
7322
7475
  sync Bring a connected source up to date with its bucket
@@ -7349,6 +7502,7 @@ Usage:
7349
7502
  remy-admin datasources create --source <slug> [--name <name>] [--placement <resource-id|shared>] [rebuild settings...]
7350
7503
  remy-admin datasources config [--source <slug>] [--placement <resource-id|shared>] [settings...]
7351
7504
  remy-admin datasources move [--source <slug>] --to <resource-id|shared> [--chains <n>] [--restart] [--wait] [--timeout <sec>]
7505
+ remy-admin datasources add-many [--source <slug>] [--budget <dollars>] [--wait] [--timeout <sec>] <file.jsonl>
7352
7506
  remy-admin datasources jobs start|list|status|approve|pause|resume|retry|cancel \u2026 (see below)
7353
7507
  remy-admin datasources connect|sync|connector|disconnect \u2026 (see below)
7354
7508
  remy-admin datasources hydrate|reindex|sample \u2026 (see below)
@@ -7486,9 +7640,12 @@ Notes:
7486
7640
  Exits ${EXIT.buildFailed} if a document failed, ${EXIT.timeout} on timeout.
7487
7641
  Re-adding an unchanged file is free: no upload, no re-embedding. Re-adding
7488
7642
  with a different --metadata updates the tags in place, also free.
7489
- add stops at the app's documentLimit, the most documents a source may hold
7490
- (shown by \`list\`): past it, add is refused with document_limit_reached
7491
- (422), and support raises the app's limit. Bulk jobs aren't held to it.
7643
+ add is refused with data_source_full (422) once a source holds what its
7644
+ placement does: a shared-pool source at the per-source cap moves to
7645
+ dedicated capacity (\`move\`), a dedicated one is resized. The message
7646
+ says which.
7647
+ add is for a file at a time. For records rendered to text in any number,
7648
+ use add-many; for files in a store or behind URLs, a job.
7492
7649
  --metadata tags documents for search-time filtering (scalars, \u226416 keys).
7493
7650
  --entities names what a document is about, as your own ids (\u2264256, no spaces
7494
7651
  or commas); an empty --entities clears them. Also free on a re-add.
@@ -7528,6 +7685,9 @@ Examples:
7528
7685
  remy-admin datasources jobs start --source archive --store raw --prefix 2024/ --wait
7529
7686
  remy-admin datasources jobs approve <id> --wait
7530
7687
 
7688
+ # Add records rendered to text, in any number: one job, priced first
7689
+ remy-admin datasources add-many --source claims --wait claims.jsonl
7690
+
7531
7691
  # Follow a customer's bucket: secrets by name, first sync stops for approval
7532
7692
  remy-admin secrets set ARCHIVE_S3_KEY --prod <value> # never on a command line you share
7533
7693
  remy-admin datasources connect --source archive --bucket acme-docs --region us-east-1 --prefix contracts/ --access-key-secret ARCHIVE_S3_KEY --secret-key-secret ARCHIVE_S3_SECRET --budget-per-sync 5
@@ -54,6 +54,10 @@ export type DataSourceJobSource = {
54
54
  */
55
55
  type: 'object';
56
56
  ref: string;
57
+ } | {
58
+ /** The documents a `datasources add-many` (or an app's `addMany()`) gave, and how many. */
59
+ type: 'documents';
60
+ documents: number;
57
61
  };
58
62
  /**
59
63
  * planning → planned → running ⇄ paused → done | failed | cancelled.
@@ -272,6 +276,25 @@ export interface DataSourceJobGetResult {
272
276
  export interface DataSourceJobResult {
273
277
  job: DataSourceJob;
274
278
  }
279
+ /** POST /datasources/add-many/open */
280
+ export interface DataSourceAddManyOpenResult {
281
+ /** Names the upload on `add-many/part` and `add-many/start`. */
282
+ uploadId: string;
283
+ }
284
+ /** POST /datasources/add-many/part */
285
+ export interface DataSourceAddManyPartResult {
286
+ /** The part's number, which `add-many/start` lists it by. */
287
+ part: number;
288
+ uploadUrl: string;
289
+ uploadFields: Record<string, string>;
290
+ }
291
+ /** What `addMany` answers with. */
292
+ export interface DataSourceAddManyResult {
293
+ /** How many documents were given. */
294
+ documentCount: number;
295
+ /** The job registering and building them; null only when none were given. */
296
+ job: DataSourceJob | null;
297
+ }
275
298
  /** POST /datasources/jobs/manifest-token */
276
299
  export interface DataSourceJobManifestTokenResult {
277
300
  key: string;
@@ -352,11 +352,6 @@ export interface DataSourcesListEntry {
352
352
  name: string | null;
353
353
  /** Total document count (active pipeline). */
354
354
  documentCount: number;
355
- /**
356
- * The most documents the source may hold, the app's limit; null when
357
- * unlimited. An add past it is refused with `document_limit_reached`.
358
- */
359
- documentLimit: number | null;
360
355
  counts: DataSourcesBuildProgress;
361
356
  /** Source-document bytes (does not include vector storage). */
362
357
  storageBytes: number;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@madewithremy/admin",
3
- "version": "0.1.67",
3
+ "version": "0.1.68",
4
4
  "description": "Admin SDK + CLI for managing production Remy apps",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",