@romedb/client 0.2.14 → 0.2.16

package/README.md

@@ -4,9 +4,9 @@
 serve --directory` or RomeDB Cloud. It runs in Node.js, Cloudflare Workers, browsers
 and any other runtime with `fetch`.
 
-The package is about 30 KB to download, has no dependencies and has no native addon. It holds the
-same code as `@romedb/node/client` and `@romedb/node/values`, byte for byte, at the same
-version. To open a local database file, use [`@romedb/node`](https://www.npmjs.com/package/@romedb/node).
+The package is about 30 KB to download, has no dependencies and has no native addon. To
+open a local database file, use [`@romedb/node`](https://www.npmjs.com/package/@romedb/node),
+which depends on this package.
 
 ## Install
 
@@ -52,10 +52,16 @@ export default {
 
 ## Using it with `@romedb/node`
 
-Each package has its own value classes. A `Record` from `@romedb/client` is not an
-instance of the `Record` from `@romedb/node/client`, so a client refuses a value from
-the other package. Use one package for all your client code, or convert values at the
-boundary.
+`@romedb/node` depends on this package at the same version. Its `client`, `values` and
+`embedded-client` entries use the classes from here, so a `Record` or `Float64` made by
+one package works in the other.
+
+That needs one installed copy of `@romedb/client`, so keep both packages at the same
+version. If the versions differ, your package manager installs a second copy, and a
+client refuses values made by the other copy with `unsupported parameter value`.
+
+The `Database` and `AsyncDatabase` classes of the main `@romedb/node` entry keep their
+own `Record` and `Vector`. Use the types of that entry with them.
 
 ## Saved requests
 

package/client.cjs

@@ -161,12 +161,14 @@ class RemoteDatabase {
   async exportFile(options = {}) {
     this.#assertOpen();
     if (this.#transport) throw new ClientError('RDB_UNSUPPORTED', 'Export embedded databases through EmbeddedDatabase.exportFile', 'not-sent');
-    if (!plainObject(options) || Object.keys(options).some(key => !['signal', 'timeoutMs', 'emptyTables', 'newIdentity'].includes(key))) throw new TypeError('export options take signal, timeoutMs, emptyTables and newIdentity');
+    if (!plainObject(options) || Object.keys(options).some(key => !['signal', 'timeoutMs', 'waitMs', 'emptyTables', 'newIdentity'].includes(key))) throw new TypeError('export options take signal, timeoutMs, emptyTables and newIdentity');
     checkCopyOptions(options);
     const duration = timeout(options.timeoutMs ?? this.#timeout);
     if (options.signal !== undefined && !(options.signal instanceof AbortSignal)) throw new TypeError('signal must be an AbortSignal');
     if (options.signal?.aborted) throw new ClientError('RDB_CANCELLED', 'Request cancelled before dispatch', 'not-sent');
     if (this.#active.size >= 32) throw new ClientError('RDB_LIMIT', 'Client request limit reached', 'not-sent');
+    if (options.waitMs !== undefined && (!Number.isInteger(options.waitMs) || options.waitMs < 1 || options.waitMs > MAX_EXPORT_WAIT)) throw new RangeError(`waitMs must be 1..${MAX_EXPORT_WAIT}`);
+    const finished = this.#cloud ? await this.#exportJob(options, duration) : undefined;
     const controller = new AbortController();
     const abort = () => controller.abort();
     options.signal?.addEventListener('abort', abort, { once: true });
@@ -183,7 +185,7 @@ class RemoteDatabase {
     const credential = this.#token;
     let status;
     try {
-      const response = await this.#fetch(this.#url + '/file' + fileQuery(options), {
+      const response = await this.#fetch(this.#url + (finished ?? '/file' + fileQuery(options)), {
         method: 'GET',
         headers: { authorization: `Bearer ${credential}`, accept: `${DATABASE_FILE_TYPE}, application/json`, ...(this.#id === undefined ? {} : { 'x-romedb-database-id': this.#id }) },
         redirect: 'manual', signal: controller.signal,
@@ -215,6 +217,53 @@ class RemoteDatabase {
       throw new ClientError(controller.signal.aborted ? 'RDB_CANCELLED' : status === undefined ? 'RDB_TRANSPORT' : 'RDB_PROTOCOL', 'Database file request failed', 'read-failed', { ...(status === undefined ? {} : { status }), cause: error });
     }
   }
+  // Cloud builds a file in a job that outlasts any one request, so this starts the job
+  // and waits for it. It gives the path of the finished file, or undefined from a
+  // service without jobs, whose `/file` builds the copy while the request waits.
+  async #exportJob(options, duration) {
+    const controller = new AbortController();
+    const abort = () => controller.abort();
+    options.signal?.addEventListener('abort', abort, { once: true });
+    this.#active.add(controller);
+    const credential = this.#token, deadline = Date.now() + (options.waitMs ?? MAX_EXPORT_WAIT);
+    const call = (method, path) => this.#fetch(this.#url + path, {
+      method, redirect: 'manual', signal: AbortSignal.any([controller.signal, AbortSignal.timeout(duration)]),
+      headers: { authorization: `Bearer ${credential}`, accept: 'application/json', ...(this.#id === undefined ? {} : { 'x-romedb-database-id': this.#id }) },
+    });
+    const refused = async response => {
+      const retryAfterMs = retryAfter(response.headers.get('retry-after'));
+      const reply = await boundedJson(response, MAX_RESPONSE).catch(() => undefined);
+      const code = plainObject(reply?.error) && typeof reply.error.code === 'string' && /^RDB_[A-Z0-9_]{1,80}$/.test(reply.error.code) ? reply.error.code : httpCode(response.status);
+      return new ClientError(code, safeMessage(reply, credential), 'read-failed', { status: response.status, ...(retryAfterMs === undefined ? {} : { retryAfterMs }) });
+    };
+    try {
+      const started = await call('POST', '/file-exports' + fileQuery(options));
+      if ([404, 405, 503].includes(started.status) && started.headers.get('retry-after') === null) { await started.body?.cancel(); return undefined; }
+      if (started.status !== 202) throw await refused(started);
+      const job = await boundedJson(started, MAX_RESPONSE);
+      if (!plainObject(job) || typeof job.id !== 'string' || !/^[0-9a-f-]{36}$/.test(job.id)) throw new ClientError('RDB_PROTOCOL', 'Invalid file export response', 'read-failed');
+      for (let failures = 0; ;) {
+        if (Date.now() >= deadline) throw new ClientError('RDB_TIMEOUT', 'The file export is still running', 'read-failed');
+        await pause(EXPORT_POLL_MS, controller.signal);
+        const reply = await call('GET', `/file-exports/${job.id}`).catch(error => {
+          if (controller.signal.aborted || ++failures > EXPORT_POLL_FAILURES) throw error;
+        });
+        if (!reply) continue;
+        if (!reply.ok) throw await refused(reply);
+        failures = 0;
+        const state = await boundedJson(reply, MAX_RESPONSE);
+        if (state?.state === 'ready') return `/file-exports/${job.id}/file`;
+        if (state?.state === 'failed') throw new ClientError('RDB_STORAGE', typeof state.error === 'string' ? state.error : 'The file export did not finish', 'read-failed');
+        if (state?.state !== 'running') throw new ClientError('RDB_PROTOCOL', 'Invalid file export state', 'read-failed');
+      }
+    } catch (error) {
+      if (error instanceof ClientError) throw error;
+      throw new ClientError(controller.signal.aborted ? 'RDB_CANCELLED' : 'RDB_TRANSPORT', 'File export request failed', 'read-failed');
+    } finally {
+      options.signal?.removeEventListener('abort', abort);
+      this.#active.delete(controller);
+    }
+  }
   async close() {
     this.#closed = true;
     for (const controller of this.#active) controller.abort();
@@ -441,6 +490,7 @@ function encodeWorkLimits(value) {
   return Object.fromEntries(Object.keys(ceilings).filter(key => Object.hasOwn(value, key)).map(key => [key, value[key]]));
 }
 function checkSequence(value) { if (!sequence(value)) throw new RangeError('sequence requires a nonnegative safe integer below Number.MAX_SAFE_INTEGER'); }
+const MAX_EXPORT_WAIT = 3 * 3600_000, EXPORT_POLL_MS = 2000, EXPORT_POLL_FAILURES = 10;
 function timeout(value) { if (!Number.isInteger(value) || value < 1 || value > 120_000) throw new RangeError('timeoutMs must be 1..120000'); return value; }
 function byteLength(text) { return encoder.encode(text).byteLength; }
 function checkBodySize(body, queryVersion = 2) {

package/client.d.ts

@@ -13,7 +13,10 @@ export interface CopyOptions {
   /** Leave out the database ID, the sequence and the receipts: the copy is a new database. */
   newIdentity?: boolean;
 }
-export interface FileExportOptions extends RequestOptions, CopyOptions {}
+export interface FileExportOptions extends RequestOptions, CopyOptions {
+  /** How long Cloud's job may take to build the file. The default is 3 hours, the longest it runs. */
+  waitMs?: number;
+}
 export interface WorkLimits { rowsRead?: number; rowMutations?: number; vmSteps?: number; }
 export interface ReadOptions extends RequestOptions { afterSequence?: number; workLimits?: WorkLimits; }
 export interface WriteOptions extends ReadOptions { requestId?: string; ifSequence?: number; }
@@ -82,8 +85,10 @@ export class RemoteDatabase implements Client {
   /** Ask Cloud to start the database's container and open the database without running a query. */
   warm(options?: RequestOptions): Promise<WarmResult>;
   /**
-   * Download the database as a .romedb file (romedb serve and serve --directory; not Cloud yet).
-   * timeoutMs covers building the copy, until the file starts arriving; signal also stops the download.
+   * Download the database as a .romedb file. timeoutMs covers each request, until the file starts
+   * arriving; signal also stops the download. Cloud builds the file in a job, so a large database
+   * can take minutes: this waits for it, up to waitMs, then downloads it. The file stays in Cloud
+   * for a day.
    */
   exportFile(options?: FileExportOptions): Promise<DatabaseFile>;
   readonly sequence: number;

package/client.mjs

@@ -166,12 +166,14 @@ class RemoteDatabase {
   async exportFile(options = {}) {
     this.#assertOpen();
     if (this.#transport) throw new ClientError('RDB_UNSUPPORTED', 'Export embedded databases through EmbeddedDatabase.exportFile', 'not-sent');
-    if (!plainObject(options) || Object.keys(options).some(key => !['signal', 'timeoutMs', 'emptyTables', 'newIdentity'].includes(key))) throw new TypeError('export options take signal, timeoutMs, emptyTables and newIdentity');
+    if (!plainObject(options) || Object.keys(options).some(key => !['signal', 'timeoutMs', 'waitMs', 'emptyTables', 'newIdentity'].includes(key))) throw new TypeError('export options take signal, timeoutMs, emptyTables and newIdentity');
     checkCopyOptions(options);
     const duration = timeout(options.timeoutMs ?? this.#timeout);
     if (options.signal !== undefined && !(options.signal instanceof AbortSignal)) throw new TypeError('signal must be an AbortSignal');
     if (options.signal?.aborted) throw new ClientError('RDB_CANCELLED', 'Request cancelled before dispatch', 'not-sent');
     if (this.#active.size >= 32) throw new ClientError('RDB_LIMIT', 'Client request limit reached', 'not-sent');
+    if (options.waitMs !== undefined && (!Number.isInteger(options.waitMs) || options.waitMs < 1 || options.waitMs > MAX_EXPORT_WAIT)) throw new RangeError(`waitMs must be 1..${MAX_EXPORT_WAIT}`);
+    const finished = this.#cloud ? await this.#exportJob(options, duration) : undefined;
     const controller = new AbortController();
     const abort = () => controller.abort();
     options.signal?.addEventListener('abort', abort, { once: true });
@@ -188,7 +190,7 @@ class RemoteDatabase {
     const credential = this.#token;
     let status;
     try {
-      const response = await this.#fetch(this.#url + '/file' + fileQuery(options), {
+      const response = await this.#fetch(this.#url + (finished ?? '/file' + fileQuery(options)), {
         method: 'GET',
         headers: { authorization: `Bearer ${credential}`, accept: `${DATABASE_FILE_TYPE}, application/json`, ...(this.#id === undefined ? {} : { 'x-romedb-database-id': this.#id }) },
         redirect: 'manual', signal: controller.signal,
@@ -220,6 +222,53 @@ class RemoteDatabase {
       throw new ClientError(controller.signal.aborted ? 'RDB_CANCELLED' : status === undefined ? 'RDB_TRANSPORT' : 'RDB_PROTOCOL', 'Database file request failed', 'read-failed', { ...(status === undefined ? {} : { status }), cause: error });
     }
   }
+  // Cloud builds a file in a job that outlasts any one request, so this starts the job
+  // and waits for it. It gives the path of the finished file, or undefined from a
+  // service without jobs, whose `/file` builds the copy while the request waits.
+  async #exportJob(options, duration) {
+    const controller = new AbortController();
+    const abort = () => controller.abort();
+    options.signal?.addEventListener('abort', abort, { once: true });
+    this.#active.add(controller);
+    const credential = this.#token, deadline = Date.now() + (options.waitMs ?? MAX_EXPORT_WAIT);
+    const call = (method, path) => this.#fetch(this.#url + path, {
+      method, redirect: 'manual', signal: AbortSignal.any([controller.signal, AbortSignal.timeout(duration)]),
+      headers: { authorization: `Bearer ${credential}`, accept: 'application/json', ...(this.#id === undefined ? {} : { 'x-romedb-database-id': this.#id }) },
+    });
+    const refused = async response => {
+      const retryAfterMs = retryAfter(response.headers.get('retry-after'));
+      const reply = await boundedJson(response, MAX_RESPONSE).catch(() => undefined);
+      const code = plainObject(reply?.error) && typeof reply.error.code === 'string' && /^RDB_[A-Z0-9_]{1,80}$/.test(reply.error.code) ? reply.error.code : httpCode(response.status);
+      return new ClientError(code, safeMessage(reply, credential), 'read-failed', { status: response.status, ...(retryAfterMs === undefined ? {} : { retryAfterMs }) });
+    };
+    try {
+      const started = await call('POST', '/file-exports' + fileQuery(options));
+      if ([404, 405, 503].includes(started.status) && started.headers.get('retry-after') === null) { await started.body?.cancel(); return undefined; }
+      if (started.status !== 202) throw await refused(started);
+      const job = await boundedJson(started, MAX_RESPONSE);
+      if (!plainObject(job) || typeof job.id !== 'string' || !/^[0-9a-f-]{36}$/.test(job.id)) throw new ClientError('RDB_PROTOCOL', 'Invalid file export response', 'read-failed');
+      for (let failures = 0; ;) {
+        if (Date.now() >= deadline) throw new ClientError('RDB_TIMEOUT', 'The file export is still running', 'read-failed');
+        await pause(EXPORT_POLL_MS, controller.signal);
+        const reply = await call('GET', `/file-exports/${job.id}`).catch(error => {
+          if (controller.signal.aborted || ++failures > EXPORT_POLL_FAILURES) throw error;
+        });
+        if (!reply) continue;
+        if (!reply.ok) throw await refused(reply);
+        failures = 0;
+        const state = await boundedJson(reply, MAX_RESPONSE);
+        if (state?.state === 'ready') return `/file-exports/${job.id}/file`;
+        if (state?.state === 'failed') throw new ClientError('RDB_STORAGE', typeof state.error === 'string' ? state.error : 'The file export did not finish', 'read-failed');
+        if (state?.state !== 'running') throw new ClientError('RDB_PROTOCOL', 'Invalid file export state', 'read-failed');
+      }
+    } catch (error) {
+      if (error instanceof ClientError) throw error;
+      throw new ClientError(controller.signal.aborted ? 'RDB_CANCELLED' : 'RDB_TRANSPORT', 'File export request failed', 'read-failed');
+    } finally {
+      options.signal?.removeEventListener('abort', abort);
+      this.#active.delete(controller);
+    }
+  }
   async close() {
     this.#closed = true;
     for (const controller of this.#active) controller.abort();
@@ -446,6 +495,7 @@ function encodeWorkLimits(value) {
   return Object.fromEntries(Object.keys(ceilings).filter(key => Object.hasOwn(value, key)).map(key => [key, value[key]]));
 }
 function checkSequence(value) { if (!sequence(value)) throw new RangeError('sequence requires a nonnegative safe integer below Number.MAX_SAFE_INTEGER'); }
+const MAX_EXPORT_WAIT = 3 * 3600_000, EXPORT_POLL_MS = 2000, EXPORT_POLL_FAILURES = 10;
 function timeout(value) { if (!Number.isInteger(value) || value < 1 || value > 120_000) throw new RangeError('timeoutMs must be 1..120000'); return value; }
 function byteLength(text) { return encoder.encode(text).byteLength; }
 function checkBodySize(body, queryVersion = 2) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@romedb/client",
-  "version": "0.2.14",
+  "version": "0.2.16",
   "description": "RomeDB client for Node.js, Cloudflare Workers and browsers. Connects by URL to romedb serve or RomeDB Cloud, with no native addon.",
   "main": "client.cjs",
   "types": "client.d.ts",
@@ -15,6 +15,9 @@
       "node": "./values.cjs",
       "default": "./values.mjs"
     },
+    "./internal": {
+      "node": "./client-internal.cjs"
+    },
     "./package.json": "./package.json"
   },
   "files": [