@lossless.org/client 1.5.1 → 1.7.0

package/ts/nosqldb/classes.collection.ts

@@ -39,6 +39,7 @@ import {
   getExpectedCollectionTopologyForSchema,
   getIdentityIndexName,
   isIdentityIndexFor,
+  isPlainIndexDocument,
   compareSmartdataTopologyStrings,
 } from './classes.collectiontopology.js';
 
@@ -210,6 +211,8 @@ export interface ISmartdataIndexInfo {
   unique: boolean;
   sparse: boolean;
   expireAfterSeconds?: number;
+  /** The filter of a partial index exactly as the backend lists it; absent otherwise. */
+  partialFilterExpression?: Record<string, unknown>;
 }
 interface ISmartdataDecoratorMetadata {
   globalSaveableProperties?: string[];
@@ -2814,6 +2817,9 @@ export class SmartdataCollection<T> {
           typeof indexArg.expireAfterSeconds === 'number'
             ? indexArg.expireAfterSeconds
             : undefined,
+        ...(isPlainIndexDocument(indexArg.partialFilterExpression)
+          ? { partialFilterExpression: indexArg.partialFilterExpression }
+          : {}),
       };
     });
   }

package/ts/nosqldb/classes.collectiontopology.ts

@@ -33,6 +33,13 @@ export interface ISmartdataCollectionTopologyIndex {
   readonly unique: boolean;
   readonly sparse: boolean;
   readonly expireAfterSeconds: number | null;
+  /**
+   * The filter of a partial index exactly as the backend lists it, present
+   * only on a partial index. Plain documents and arrays inside it are frozen
+   * copies; BSON values keep the type the driver decoded. A model never
+   * declares one, so an expected index never carries it.
+   */
+  readonly partialFilterExpression?: Readonly<Record<string, unknown>>;
 }
 
 export interface ISmartdataExpectedCollectionTopology {
@@ -110,6 +117,7 @@ const ordinaryActualIndexKeys = new Set([
   'unique',
   'sparse',
   'expireAfterSeconds',
+  'partialFilterExpression',
 ]);
 const textActualIndexKeys = new Set([
   ...ordinaryActualIndexKeys,
@@ -141,6 +149,37 @@ export const compareSmartdataTopologyStrings = (
   return leftCodePoints.length < rightCodePoints.length ? -1 : 1;
 };
 
+/** @internal Whether a listed index property is a plain BSON document. */
+export const isPlainIndexDocument = (
+  valueArg: unknown,
+): valueArg is Record<string, unknown> => {
+  if (typeof valueArg !== 'object' || valueArg === null || Array.isArray(valueArg)) {
+    return false;
+  }
+  const prototype = Object.getPrototypeOf(valueArg);
+  return prototype === Object.prototype || prototype === null;
+};
+
+/**
+ * Copies and freezes the documents and arrays of a listed index filter, so the
+ * reported filter is exactly what the backend listed and nobody can edit it
+ * afterwards. BSON values — ObjectIds, dates, decimals — are returned as the
+ * driver decoded them.
+ */
+const snapshotIndexFilterValue = (valueArg: unknown): unknown => {
+  if (Array.isArray(valueArg)) {
+    return Object.freeze(valueArg.map((entryArg) => snapshotIndexFilterValue(entryArg)));
+  }
+  if (isPlainIndexDocument(valueArg)) {
+    const copy: Record<string, unknown> = {};
+    for (const [key, entry] of Object.entries(valueArg)) {
+      copy[key] = snapshotIndexFilterValue(entry);
+    }
+    return Object.freeze(copy);
+  }
+  return valueArg;
+};
+
 function freezeIndex(
   indexArg: {
     name: string;
@@ -152,6 +191,7 @@ function freezeIndex(
     unique: boolean;
     sparse: boolean;
     expireAfterSeconds: number | null;
+    partialFilterExpression?: Readonly<Record<string, unknown>>;
   },
 ): ISmartdataCollectionTopologyIndex {
   return Object.freeze({
@@ -162,6 +202,15 @@ function freezeIndex(
     unique: indexArg.unique,
     sparse: indexArg.sparse,
     expireAfterSeconds: indexArg.expireAfterSeconds,
+    // Absent rather than null on an ordinary index, so every index that is
+    // not partial reads exactly as it did before filters were reported.
+    ...(indexArg.partialFilterExpression === undefined
+      ? {}
+      : {
+          partialFilterExpression: snapshotIndexFilterValue(
+            indexArg.partialFilterExpression,
+          ) as Readonly<Record<string, unknown>>,
+        }),
   });
 }
 
@@ -390,6 +439,13 @@ const normalizeActualIndex = (
   if (rawIndex.sparse !== undefined && typeof rawIndex.sparse !== 'boolean') {
     supported = false;
   }
+  const partialFilterExpression = rawIndex.partialFilterExpression;
+  if (
+    partialFilterExpression !== undefined &&
+    !isPlainIndexDocument(partialFilterExpression)
+  ) {
+    supported = false;
+  }
   const expireAfterSeconds = rawIndex.expireAfterSeconds;
   if (
     expireAfterSeconds !== undefined &&
@@ -495,6 +551,9 @@ const normalizeActualIndex = (
       expireAfterSeconds >= 0
         ? expireAfterSeconds
         : null,
+    ...(isPlainIndexDocument(partialFilterExpression)
+      ? { partialFilterExpression }
+      : {}),
   });
   if (
     name === '_id_' &&
@@ -504,7 +563,8 @@ const normalizeActualIndex = (
     normalized.keys[0].weight === null &&
     (rawIndex.unique === undefined || rawIndex.unique === true) &&
     normalized.sparse === false &&
-    normalized.expireAfterSeconds === null
+    normalized.expireAfterSeconds === null &&
+    normalized.partialFilterExpression === undefined
   ) {
     normalized = fixedIdIndex;
   }

package/ts/nosqldb/classes.db.ts

@@ -37,6 +37,35 @@ import type { ICapabilities } from '../core/interfaces.js';
  */
 export type TConnectionStatus = 'initial' | 'disconnected' | 'connected' | 'failed';
 
+/**
+ * How a `SmartdataDb` connects. The first four fields locate and authenticate
+ * the database — the same shape as `IMongoDescriptor`, so every descriptor is
+ * valid options — and the pool fields tune the driver's connection pool.
+ */
+export interface ISmartdataDbOptions {
+  /**
+   * The connection URL. `<USERNAME>`/`<USER>`, `<PASSWORD>` and `<DBNAME>`
+   * (upper or lower case) are replaced with the URL-encoded `mongoDbUser`,
+   * `mongoDbPass` and `mongoDbName`.
+   */
+  mongoDbUrl: string;
+  /** The database to use. */
+  mongoDbName?: string;
+  mongoDbUser?: string;
+  mongoDbPass?: string;
+  /** Upper bound of pooled connections per server. Default 100. */
+  maxPoolSize?: number;
+  /** Milliseconds an idle pooled connection is kept before it is closed. Default 300000. */
+  maxIdleTimeMS?: number;
+  /**
+   * Milliseconds to wait for a suitable server. Default 30000. A bounded
+   * `init({ timeoutMs })` replaces it with the remaining init budget.
+   */
+  serverSelectionTimeoutMS?: number;
+  /** Milliseconds a socket may stay inactive before it is closed. Default 30000. */
+  socketTimeoutMS?: number;
+}
+
 export interface ISmartdataReadinessOptions {
   timeoutMs?: number;
   signal?: AbortSignal;
@@ -138,7 +167,7 @@ const assertExactDataObject = (
 };
 
 export class SmartdataDb {
-  smartdataOptions: plugins.tsclass.database.IMongoDescriptor;
+  smartdataOptions: ISmartdataDbOptions;
   mongoDbClient!: plugins.mongodb.MongoClient;
   mongoDb!: plugins.mongodb.Db;
   status: TConnectionStatus;
@@ -151,7 +180,7 @@ export class SmartdataDb {
   private engineIdentityState: ISmartdataEngineIdentity = unidentifiedSmartdataEngine;
   private capabilitiesState: ICapabilities | undefined;
 
-  constructor(smartdataOptions: plugins.tsclass.database.IMongoDescriptor) {
+  constructor(smartdataOptions: ISmartdataDbOptions) {
     this.smartdataOptions = smartdataOptions;
     this.status = 'initial';
     void this.statusConnectedDeferred.promise.catch(() => {});
@@ -219,9 +248,7 @@ export class SmartdataDb {
           .replace('<DBNAME>', this.smartdataOptions.mongoDbName || '')
           .replace('<dbname>', this.smartdataOptions.mongoDbName || '');
 
-        const descriptor = this.smartdataOptions as plugins.tsclass.database.IMongoDescriptor &
-          Pick<plugins.mongodb.MongoClientOptions, 'maxPoolSize' | 'maxIdleTimeMS' |
-            'serverSelectionTimeoutMS' | 'socketTimeoutMS'>;
+        const descriptor = this.smartdataOptions;
         const clientOptions: plugins.mongodb.MongoClientOptions = {
           maxPoolSize: descriptor.maxPoolSize ?? 100,
           maxIdleTimeMS: descriptor.maxIdleTimeMS ?? 300000,

package/ts/nosqldb/classes.doc.ts

@@ -5774,9 +5774,11 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
   }
 
   /**
-   * saves this instance (optionally within a transaction)
+   * Saves this instance, optionally within a transaction. `session` takes the
+   * handle from `db.createSession()` — leased for the one write, like every
+   * other model API — or a raw driver session.
    */
-  public async save(opts?: { session?: plugins.mongodb.ClientSession }) {
+  public async save(opts?: { session?: TSmartdataOrdinarySession }) {
     if (getOrdinaryPersistencePolicy(this.constructor)) {
       throw new SmartdataPersistenceError('unsupported_operation',
         'Validated ordinary models use insert() or transactional postimage updates.');
@@ -5980,9 +5982,11 @@ export class SmartDataDbDoc<T extends TImplements, TImplements, TManager extends
   }
 
   /**
-   * deletes a document from the database (optionally within a transaction)
+   * Deletes this instance's document, optionally within a transaction.
+   * `session` takes the handle from `db.createSession()` — leased for the one
+   * write, like every other model API — or a raw driver session.
    */
-  public async delete(opts?: { session?: plugins.mongodb.ClientSession }) {
+  public async delete(opts?: { session?: TSmartdataOrdinarySession }) {
     const exactPersistencePolicy = (this.constructor as any)[
       exactPersistencePolicySymbol
     ] as { forbiddenDelete?: () => Promise<never> } | undefined;

package/ts/objectstorage/classes.bucket.ts

@@ -30,9 +30,171 @@ export interface IListObjectKeysPageResult {
   nextStartAfter?: string;
 }
 
+export interface IListObjectEntriesPageOptions extends IListObjectKeysPageOptions {
+  /** Optional caller-owned cancellation signal */
+  signal?: AbortSignal;
+}
+
+/**
+ * One object as ListObjectsV2 reported it when the page was listed.
+ * - `size`: the `Size` field, the object's byte length; HEAD reports the same
+ *   value as `ContentLength`.
+ * - `etag`: the `ETag` field without its surrounding double quotes; HEAD
+ *   reports the same entity tag, quoted. It identifies an object version, it
+ *   is not a content digest: AWS S3 derives it from the content MD5 only for
+ *   single-part uploads without SSE-KMS or SSE-C, and other providers define
+ *   it themselves.
+ * - `lastModified`: the `LastModified` field when the provider sends one.
+ */
+export interface IObjectListingEntry {
+  key: string;
+  size: number;
+  etag: string;
+  lastModified?: Date;
+}
+
+export interface IListObjectEntriesPageResult {
+  entries: IObjectListingEntry[];
+  nextStartAfter?: string;
+}
+
+export interface IListAllObjectEntriesOptions {
+  /** Entries fetched per ListObjectsV2 request, 1 to 1000 (default 1000) */
+  pageSize?: number;
+  /** Optional caller-owned cancellation signal */
+  signal?: AbortSignal;
+}
+
 const compareUtf8ObjectKeys = (leftArg: string, rightArg: string): number =>
   Buffer.compare(Buffer.from(leftArg, 'utf8'), Buffer.from(rightArg, 'utf8'));
 
+const assertListingPageOptions = (
+  optionsArg: IListObjectKeysPageOptions,
+  labelArg: string,
+): string => {
+  if (!optionsArg || typeof optionsArg !== 'object' || Array.isArray(optionsArg)) {
+    throw new TypeError(`${labelArg} options must be a plain object`);
+  }
+  const prefix = optionsArg.prefix ?? '';
+  if (typeof prefix !== 'string') {
+    throw new TypeError(`${labelArg} prefix must be a string`);
+  }
+  if (!Number.isInteger(optionsArg.limit) || optionsArg.limit < 1 || optionsArg.limit > 1_000) {
+    throw new Error(`${labelArg} limit must be an integer from 1 to 1000`);
+  }
+  if (
+    optionsArg.startAfter !== undefined
+    && (
+      typeof optionsArg.startAfter !== 'string'
+      || optionsArg.startAfter.length === 0
+      || !optionsArg.startAfter.startsWith(prefix)
+    )
+  ) {
+    throw new Error(`${labelArg} startAfter must be a non-empty key under the prefix`);
+  }
+  return prefix;
+};
+
+const createListingPageCommand = (
+  bucketNameArg: string,
+  prefixArg: string,
+  optionsArg: IListObjectKeysPageOptions,
+): plugins.s3.ListObjectsV2Command => new plugins.s3.ListObjectsV2Command({
+  Bucket: bucketNameArg,
+  Prefix: prefixArg,
+  MaxKeys: optionsArg.limit,
+  ...(optionsArg.startAfter ? { StartAfter: optionsArg.startAfter } : {}),
+});
+
+/**
+ * Validate one ListObjectsV2 keyset page: a truncation flag, at most `limit`
+ * string keys under the prefix in strict UTF-8 byte order after `startAfter`,
+ * and no empty truncated page.
+ */
+const readListingPageContents = (
+  responseArg: plugins.s3.ListObjectsV2Output,
+  prefixArg: string,
+  optionsArg: IListObjectKeysPageOptions,
+  labelArg: string,
+): { contents: plugins.s3._Object[]; keys: string[]; nextStartAfter?: string } => {
+  if (typeof responseArg.IsTruncated !== 'boolean') {
+    throw new Error(`Object storage returned an invalid ${labelArg} truncation flag`);
+  }
+  if (responseArg.Contents !== undefined && !Array.isArray(responseArg.Contents)) {
+    throw new Error(`Object storage returned invalid ${labelArg} contents`);
+  }
+  const contents = responseArg.Contents ?? [];
+  const keys = contents.map((entry) => {
+    if (!entry || typeof entry.Key !== 'string') {
+      throw new Error(`Object storage returned a ${labelArg} entry without a string key`);
+    }
+    return entry.Key;
+  });
+  if (keys.length > optionsArg.limit) {
+    throw new Error(`Object storage returned more ${labelArg === 'key page' ? 'keys' : 'entries'} than the requested page limit`);
+  }
+  if (new Set(keys).size !== keys.length) {
+    throw new Error('Object storage returned duplicate keys in one page');
+  }
+
+  let previousKey = optionsArg.startAfter;
+  for (const key of keys) {
+    if (!key.startsWith(prefixArg)) {
+      throw new Error('Object storage returned a key outside the requested prefix');
+    }
+    if (previousKey !== undefined && compareUtf8ObjectKeys(key, previousKey) <= 0) {
+      throw new Error('Object storage returned keys outside strict UTF-8 byte order');
+    }
+    previousKey = key;
+  }
+  if (responseArg.IsTruncated && keys.length === 0) {
+    throw new Error(`Object storage returned an empty truncated ${labelArg}`);
+  }
+  return {
+    contents,
+    keys,
+    ...(responseArg.IsTruncated ? { nextStartAfter: keys[keys.length - 1] } : {}),
+  };
+};
+
+/**
+ * Strip the double quotes S3 puts around an entity tag. A provider that
+ * returns the tag unquoted is accepted as is; anything else is refused.
+ */
+const normalizeListingEtag = (etagArg: unknown, keyArg: string): string => {
+  if (typeof etagArg !== 'string') {
+    throw new Error(`Object storage returned a listing entry without an ETag for key '${keyArg}'`);
+  }
+  const quoted = /^"([^"]+)"$/.exec(etagArg);
+  const etag = quoted ? quoted[1] : etagArg;
+  if (etag.length === 0 || etag.includes('"')) {
+    throw new Error(`Object storage returned an invalid ETag for key '${keyArg}'`);
+  }
+  return etag;
+};
+
+const toObjectListingEntry = (
+  entryArg: plugins.s3._Object,
+  keyArg: string,
+): IObjectListingEntry => {
+  if (!Number.isSafeInteger(entryArg.Size) || (entryArg.Size as number) < 0) {
+    throw new Error(`Object storage returned a listing entry without a valid size for key '${keyArg}'`);
+  }
+  const lastModified = entryArg.LastModified;
+  if (
+    lastModified !== undefined
+    && (!(lastModified instanceof Date) || Number.isNaN(lastModified.getTime()))
+  ) {
+    throw new Error(`Object storage returned an invalid LastModified for key '${keyArg}'`);
+  }
+  return {
+    key: keyArg,
+    size: entryArg.Size as number,
+    etag: normalizeListingEtag(entryArg.ETag, keyArg),
+    ...(lastModified ? { lastModified } : {}),
+  };
+};
+
 const destroyReadableAndWaitForSettlement = async (
   readableArg: plugins.stream.Readable,
 ): Promise<void> => {
@@ -933,73 +1095,90 @@ export class Bucket {
   public async listObjectKeysPage(
     optionsArg: IListObjectKeysPageOptions,
   ): Promise<IListObjectKeysPageResult> {
-    if (!optionsArg || typeof optionsArg !== 'object' || Array.isArray(optionsArg)) {
-      throw new TypeError('Object key page options must be a plain object');
-    }
-    const prefix = optionsArg.prefix ?? '';
-    if (typeof prefix !== 'string') {
-      throw new TypeError('Object key page prefix must be a string');
-    }
-    if (!Number.isInteger(optionsArg.limit) || optionsArg.limit < 1 || optionsArg.limit > 1_000) {
-      throw new Error('Object key page limit must be an integer from 1 to 1000');
-    }
-    if (
-      optionsArg.startAfter !== undefined
-      && (
-        typeof optionsArg.startAfter !== 'string'
-        || optionsArg.startAfter.length === 0
-        || !optionsArg.startAfter.startsWith(prefix)
-      )
-    ) {
-      throw new Error('Object key page startAfter must be a non-empty key under the prefix');
-    }
-
+    const prefix = assertListingPageOptions(optionsArg, 'Object key page');
     const response = await this.smartbucketRef.storageClient.send(
-      new plugins.s3.ListObjectsV2Command({
-        Bucket: this.name,
-        Prefix: prefix,
-        MaxKeys: optionsArg.limit,
-        ...(optionsArg.startAfter ? { StartAfter: optionsArg.startAfter } : {}),
-      }),
+      createListingPageCommand(this.name, prefix, optionsArg),
     );
-    if (typeof response.IsTruncated !== 'boolean') {
-      throw new Error('Object storage returned an invalid key page truncation flag');
-    }
-    if (response.Contents !== undefined && !Array.isArray(response.Contents)) {
-      throw new Error('Object storage returned invalid key page contents');
-    }
+    const { keys, nextStartAfter } = readListingPageContents(
+      response,
+      prefix,
+      optionsArg,
+      'key page',
+    );
+    return {
+      keys,
+      ...(nextStartAfter !== undefined ? { nextStartAfter } : {}),
+    };
+  }
 
-    const keys = (response.Contents ?? []).map((entry) => {
-      if (!entry || typeof entry.Key !== 'string') {
-        throw new Error('Object storage returned a key page entry without a string key');
-      }
-      return entry.Key;
-    });
-    if (keys.length > optionsArg.limit) {
-      throw new Error('Object storage returned more keys than the requested page limit');
+  /**
+   * List one keyset page of objects with the size, ETag and last-modified time
+   * ListObjectsV2 already returns, so a caller needs no HEAD per object.
+   * Pagination, prefix, limit (1 to 1000) and `startAfter` behave exactly like
+   * `listObjectKeysPage()`: `nextStartAfter` is present only while the provider
+   * reports more entries. `signal` aborts the in-flight request and rejects with
+   * its reason. This requires a general-purpose S3 provider with UTF-8
+   * byte-ordered keys.
+   */
+  public async listObjectEntriesPage(
+    optionsArg: IListObjectEntriesPageOptions,
+  ): Promise<IListObjectEntriesPageResult> {
+    const prefix = assertListingPageOptions(optionsArg, 'Object entry page');
+    if (optionsArg.signal !== undefined && !(optionsArg.signal instanceof AbortSignal)) {
+      throw new TypeError('Object entry page signal must be an AbortSignal');
     }
-    if (new Set(keys).size !== keys.length) {
-      throw new Error('Object storage returned duplicate keys in one page');
+    const operation = beginBasicBucketOperation(this, optionsArg.signal);
+    try {
+      operation.assertCurrent();
+      const command = createListingPageCommand(operation.bucketName, prefix, optionsArg);
+      const response = await operation.runProvider(
+        () => operation.client.send(command, { abortSignal: operation.signal }),
+      );
+      operation.assertCurrent();
+      const { contents, keys, nextStartAfter } = readListingPageContents(
+        response,
+        prefix,
+        optionsArg,
+        'entry page',
+      );
+      return {
+        entries: contents.map((entry, index) => toObjectListingEntry(entry, keys[index])),
+        ...(nextStartAfter !== undefined ? { nextStartAfter } : {}),
+      };
+    } finally {
+      operation.finish();
     }
+  }
 
-    let previousKey = optionsArg.startAfter;
-    for (const key of keys) {
-      if (!key.startsWith(prefix)) {
-        throw new Error('Object storage returned a key outside the requested prefix');
-      }
-      if (previousKey !== undefined && compareUtf8ObjectKeys(key, previousKey) <= 0) {
-        throw new Error('Object storage returned keys outside strict UTF-8 byte order');
+  /**
+   * Iterate every object under a prefix with its size, ETag and last-modified
+   * time, one `listObjectEntriesPage()` request per page. No request is in
+   * flight while the caller holds an entry; breaking out stops the listing.
+   * @example
+   * ```ts
+   * for await (const entry of bucket.listAllObjectEntries('npm/', { signal })) {
+   *   console.log(entry.key, entry.size, entry.etag);
+   * }
+   * ```
+   */
+  public async *listAllObjectEntries(
+    prefix: string = '',
+    optionsArg: IListAllObjectEntriesOptions = {},
+  ): AsyncIterableIterator<IObjectListingEntry> {
+    const limit = optionsArg.pageSize ?? 1_000;
+    let startAfter: string | undefined;
+    do {
+      const page = await this.listObjectEntriesPage({
+        prefix,
+        limit,
+        ...(startAfter !== undefined ? { startAfter } : {}),
+        ...(optionsArg.signal ? { signal: optionsArg.signal } : {}),
+      });
+      for (const entry of page.entries) {
+        yield entry;
       }
-      previousKey = key;
-    }
-    if (response.IsTruncated && keys.length === 0) {
-      throw new Error('Object storage returned an empty truncated key page');
-    }
-
-    return {
-      keys,
-      ...(response.IsTruncated ? { nextStartAfter: keys[keys.length - 1] } : {}),
-    };
+      startAfter = page.nextStartAfter;
+    } while (startAfter !== undefined);
   }
 
   /**