harperdb 4.3.20 → 4.3.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harperdb",
-  "version": "4.3.20",
+  "version": "4.3.21",
   "description": "HarperDB is a distributed database, caching service, streaming broker, and application development platform focused on performance and ease of use.",
   "keywords": [
     "database",
@@ -10,10 +10,18 @@
     "broker",
     "mqtt",
     "real-time",
-    "cloud",
     "enterprise",
     "Fastify",
-    "NATS"
+    "NATS",
+    "HarperDB",
+    "Harper",
+    "clustering",
+    "replication",
+    "REST",
+    "WebSockets",
+    "decentralized",
+    "server-sent events",
+    "document store"
   ],
   "main": "harperdb.js",
   "bin": {
@@ -35,8 +43,8 @@
   },
   "license": "SEE LICENSE IN LICENSE",
   "dependencies": {
-    "@aws-sdk/client-s3": "3.535.0",
-    "@aws-sdk/lib-storage": "3.535.0",
+    "@aws-sdk/client-s3": "3.635.0",
+    "@aws-sdk/lib-storage": "3.635.0",
     "@endo/static-module-record": "^1.0.4",
     "@fastify/autoload": "5.7.1",
     "@fastify/compress": "~6.5.0",
@@ -69,7 +77,7 @@
     "json2csv": "5.0.7",
     "jsonata": "1.8.7",
     "jsonwebtoken": "9.0.2",
-    "lmdb": "3.0.12",
+    "lmdb": "3.0.14",
     "lodash": "4.17.21",
     "mathjs": "11.12.0",
     "minimist": "1.2.8",

package/resources/Resource.d.ts

@@ -20,7 +20,6 @@ export declare class Resource implements ResourceInterface {
     static transactions: Transaction[] & {
         timestamp: number;
     };
-    [CONTEXT]: Context;
     constructor(identifier: Id, source: any);
     /**
      * The get methods are for directly getting a resource, and called for HTTP GET requests.

package/resources/Table.d.ts

@@ -0,0 +1,588 @@
+/**
+ * This module provides the main table implementation of the Resource API, providing full access to HarperDB
+ * tables through the interface defined by the Resource class. This module is responsible for handling these
+ * table-level interactions, loading records, updating records, querying, and more.
+ */
+import { Database } from 'lmdb';
+import { Query, ResourceInterface, SubscriptionRequest, Id, Context } from './ResourceInterface';
+import { Resource } from './Resource';
+export interface Table {
+    primaryStore: Database;
+    auditStore: Database;
+    indices: {};
+    databasePath: string;
+    tableName: string;
+    databaseName: string;
+    attributes: any[];
+    primaryKey: string;
+    subscriptions: Map<any, Function[]>;
+    expirationMS: number;
+    indexingOperations?: Promise<void>;
+    sources: {
+        new (): ResourceInterface;
+    }[];
+    Transaction: ReturnType<typeof makeTable>;
+}
+/**
+ * This returns a Table class for the given table settings (determined from the metadata table)
+ * Instances of the returned class are Resource instances, intended to provide a consistent view or transaction of the table
+ * @param options
+ */
+export declare function makeTable(options: any): {
+    new (identifier: Id, source: any): {
+        /**
+         * This is a request to explicitly ensure that the record is loaded from source, rather than only using the local record.
+         * This will load from source if the current record is expired, missing, or invalidated.
+         * @returns
+         */
+        ensureLoaded(): any;
+        /**
+         * This retrieves the data of this resource. By default, with no argument, just return `this`.
+         * @param query - If included, specifies a query to perform on the record
+         */
+        get(query?: Query | string): Promise<object | void> | object | void;
+        /**
+         * Determine if the user is allowed to get/read data from the current resource
+         * @param user The current, authenticated user
+         * @param query The parsed query from the search part of the URL
+         */
+        allowRead(user: any, query: any): any;
+        /**
+         * Determine if the user is allowed to update data from the current resource
+         * @param user The current, authenticated user
+         * @param updated_data
+         * @param full_update
+         */
+        allowUpdate(user: any, updated_data: any): boolean;
+        /**
+         * Determine if the user is allowed to create new data in the current resource
+         * @param user The current, authenticated user
+         * @param new_data
+         */
+        allowCreate(user: any, new_data: {}): boolean;
+        /**
+         * Determine if the user is allowed to delete from the current resource
+         * @param user The current, authenticated user
+         */
+        allowDelete(user: any): any;
+        /**
+         * Start updating a record. The returned resource will record changes which are written
+         * once the corresponding transaction is committed. These changes can (eventually) include CRDT type operations.
+         * @param updates This can be a record to update the current resource with.
+         * @param full_update The provided data in updates is the full intended record; any properties in the existing record that are not in the updates, should be removed
+         */
+        update(updates?: any, full_update?: boolean): any;
+        addTo(property: any, value: any): void;
+        subtractFrom(property: any, value: any): void;
+        getMetadata(): any;
+        invalidate(options: any): void;
+        /**
+         * This is intended to acquire a lock on a record from the whole cluster.
+         */
+        lock(): void;
+        /**
+         * Store the provided record data into the current resource. This is not written
+         * until the corresponding transaction is committed. This will either immediately fail (synchronously) or always
+         * succeed. That doesn't necessarily mean it will "win", another concurrent put could come "after" (monotonically,
+         * even if not chronologically) this one.
+         * @param record
+         * @param options
+         */
+        put(record: any): void;
+        patch(record_update: any): void;
+        _writeUpdate(record_update: any, full_update: boolean, options?: any): void;
+        delete(request?: Query | string): Promise<boolean>;
+        _writeDelete(options?: any): boolean;
+        search(request: Query): AsyncIterable<any>;
+        subscribe(request: SubscriptionRequest): Promise<{
+            listener: (key: any) => any;
+            subscriptions: [];
+            startTime?: number;
+            end(): void;
+            toJSON(): {
+                name: string;
+            };
+            resolveNext: Function;
+            queue: any[];
+            hasDataListeners: boolean;
+            push(message: any): void;
+            send(message: any): void;
+            getNextMessage(): any;
+            on(event_name: any, listener: any): any;
+            [Symbol.asyncIterator](): {
+                queue: import("./IterableEventQueue").IterableEventQueue;
+                push(message: any): void;
+                next(): Promise<unknown> | {
+                    value: any;
+                };
+                return(value: any): {
+                    value: any;
+                    done: boolean;
+                };
+                throw(error: any): {
+                    done: boolean;
+                };
+            };
+            [EventEmitter.captureRejectionSymbol]?<K>(error: Error, event: string | symbol, ...args: any[]): void;
+            addListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            once<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            removeListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            off<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            removeAllListeners(eventName?: string | symbol): any;
+            setMaxListeners(n: number): any;
+            getMaxListeners(): number;
+            listeners<K>(eventName: string | symbol): Function[];
+            rawListeners<K>(eventName: string | symbol): Function[];
+            emit<K>(eventName: string | symbol, ...args: any[]): boolean;
+            listenerCount<K>(eventName: string | symbol, listener?: Function): number;
+            prependListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            prependOnceListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            eventNames(): (string | symbol)[];
+        }>;
+        doesExist(): boolean;
+        /**
+         * Publishing a message to a record adds an (observable) entry in the audit log, but does not change
+         * the record at all. This entries should be replicated and trigger subscription listeners.
+         * @param id
+         * @param message
+         * @param options
+         */
+        publish(message: any, options?: any): void;
+        _writePublish(message: any, options?: any): void;
+        validate(record: any, patch?: any): void;
+        getUpdatedTime(): any;
+        wasLoadedFromSource(): boolean | void;
+        post(new_record: any): any;
+        connect(query?: {}): AsyncIterable<any>;
+        getId(): any;
+        getContext(): Context;
+    };
+    name: any;
+    primaryStore: any;
+    auditStore: any;
+    primaryKey: any;
+    tableName: any;
+    indices: any;
+    audit: any;
+    databasePath: any;
+    databaseName: any;
+    attributes: any;
+    expirationTimer: any;
+    createdTimeProperty: any;
+    updatedTimeProperty: any;
+    propertyResolvers: any;
+    sources: any[];
+    readonly expirationMS: any;
+    dbisDB: any;
+    schemaDefined: any;
+    /**
+     * This defines a source for a table. This effectively makes a table into a cache, where the canonical
+     * source of data (or source of truth) is provided here in the Resource argument. Additional options
+     * can be provided to indicate how the caching should be handled.
+     * @param source
+     * @param options
+     * @returns
+     */
+    sourcedFrom(source: any, options: any): any;
+    /**
+     * Gets a resource instance, as defined by the Resource class, adding the table-specific handling
+     * of also loading the stored record into the resource instance.
+     * @param id
+     * @param request
+     * @param options An important option is ensureLoaded, which can be used to indicate that it is necessary for a caching table to load data from the source if there is not a local copy of the data in the table (usually not necessary for a delete, for example).
+     * @returns
+     */
+    getResource(id: Id, request: any, resource_options?: any): Promise<{
+        /**
+         * This is a request to explicitly ensure that the record is loaded from source, rather than only using the local record.
+         * This will load from source if the current record is expired, missing, or invalidated.
+         * @returns
+         */
+        ensureLoaded(): any;
+        /**
+         * This retrieves the data of this resource. By default, with no argument, just return `this`.
+         * @param query - If included, specifies a query to perform on the record
+         */
+        get(query?: Query | string): Promise<object | void> | object | void;
+        /**
+         * Determine if the user is allowed to get/read data from the current resource
+         * @param user The current, authenticated user
+         * @param query The parsed query from the search part of the URL
+         */
+        allowRead(user: any, query: any): any;
+        /**
+         * Determine if the user is allowed to update data from the current resource
+         * @param user The current, authenticated user
+         * @param updated_data
+         * @param full_update
+         */
+        allowUpdate(user: any, updated_data: any): boolean;
+        /**
+         * Determine if the user is allowed to create new data in the current resource
+         * @param user The current, authenticated user
+         * @param new_data
+         */
+        allowCreate(user: any, new_data: {}): boolean;
+        /**
+         * Determine if the user is allowed to delete from the current resource
+         * @param user The current, authenticated user
+         */
+        allowDelete(user: any): any;
+        /**
+         * Start updating a record. The returned resource will record changes which are written
+         * once the corresponding transaction is committed. These changes can (eventually) include CRDT type operations.
+         * @param updates This can be a record to update the current resource with.
+         * @param full_update The provided data in updates is the full intended record; any properties in the existing record that are not in the updates, should be removed
+         */
+        update(updates?: any, full_update?: boolean): any;
+        addTo(property: any, value: any): void;
+        subtractFrom(property: any, value: any): void;
+        getMetadata(): any;
+        invalidate(options: any): void;
+        /**
+         * This is intended to acquire a lock on a record from the whole cluster.
+         */
+        lock(): void;
+        /**
+         * Store the provided record data into the current resource. This is not written
+         * until the corresponding transaction is committed. This will either immediately fail (synchronously) or always
+         * succeed. That doesn't necessarily mean it will "win", another concurrent put could come "after" (monotonically,
+         * even if not chronologically) this one.
+         * @param record
+         * @param options
+         */
+        put(record: any): void;
+        patch(record_update: any): void;
+        _writeUpdate(record_update: any, full_update: boolean, options?: any): void;
+        delete(request?: Query | string): Promise<boolean>;
+        _writeDelete(options?: any): boolean;
+        search(request: Query): AsyncIterable<any>;
+        subscribe(request: SubscriptionRequest): Promise<{
+            listener: (key: any) => any;
+            subscriptions: [];
+            startTime?: number;
+            end(): void;
+            toJSON(): {
+                name: string;
+            };
+            resolveNext: Function;
+            queue: any[];
+            hasDataListeners: boolean;
+            push(message: any): void;
+            send(message: any): void;
+            getNextMessage(): any;
+            on(event_name: any, listener: any): any;
+            [Symbol.asyncIterator](): {
+                queue: import("./IterableEventQueue").IterableEventQueue;
+                push(message: any): void;
+                next(): Promise<unknown> | {
+                    value: any;
+                };
+                return(value: any): {
+                    value: any;
+                    done: boolean;
+                };
+                throw(error: any): {
+                    done: boolean;
+                };
+            };
+            [EventEmitter.captureRejectionSymbol]?<K>(error: Error, event: string | symbol, ...args: any[]): void;
+            addListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            once<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            removeListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            off<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            removeAllListeners(eventName?: string | symbol): any;
+            setMaxListeners(n: number): any;
+            getMaxListeners(): number;
+            listeners<K>(eventName: string | symbol): Function[];
+            rawListeners<K>(eventName: string | symbol): Function[];
+            emit<K>(eventName: string | symbol, ...args: any[]): boolean;
+            listenerCount<K>(eventName: string | symbol, listener?: Function): number;
+            prependListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            prependOnceListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            eventNames(): (string | symbol)[];
+        }>;
+        doesExist(): boolean;
+        /**
+         * Publishing a message to a record adds an (observable) entry in the audit log, but does not change
+         * the record at all. This entries should be replicated and trigger subscription listeners.
+         * @param id
+         * @param message
+         * @param options
+         */
+        publish(message: any, options?: any): void;
+        _writePublish(message: any, options?: any): void;
+        validate(record: any, patch?: any): void;
+        getUpdatedTime(): any;
+        wasLoadedFromSource(): boolean | void;
+        post(new_record: any): any;
+        connect(query?: {}): AsyncIterable<any>;
+        getId(): any;
+        getContext(): Context;
+    }> | {
+        /**
+         * This is a request to explicitly ensure that the record is loaded from source, rather than only using the local record.
+         * This will load from source if the current record is expired, missing, or invalidated.
+         * @returns
+         */
+        ensureLoaded(): any;
+        /**
+         * This retrieves the data of this resource. By default, with no argument, just return `this`.
+         * @param query - If included, specifies a query to perform on the record
+         */
+        get(query?: Query | string): Promise<object | void> | object | void;
+        /**
+         * Determine if the user is allowed to get/read data from the current resource
+         * @param user The current, authenticated user
+         * @param query The parsed query from the search part of the URL
+         */
+        allowRead(user: any, query: any): any;
+        /**
+         * Determine if the user is allowed to update data from the current resource
+         * @param user The current, authenticated user
+         * @param updated_data
+         * @param full_update
+         */
+        allowUpdate(user: any, updated_data: any): boolean;
+        /**
+         * Determine if the user is allowed to create new data in the current resource
+         * @param user The current, authenticated user
+         * @param new_data
+         */
+        allowCreate(user: any, new_data: {}): boolean;
+        /**
+         * Determine if the user is allowed to delete from the current resource
+         * @param user The current, authenticated user
+         */
+        allowDelete(user: any): any;
+        /**
+         * Start updating a record. The returned resource will record changes which are written
+         * once the corresponding transaction is committed. These changes can (eventually) include CRDT type operations.
+         * @param updates This can be a record to update the current resource with.
+         * @param full_update The provided data in updates is the full intended record; any properties in the existing record that are not in the updates, should be removed
+         */
+        update(updates?: any, full_update?: boolean): any;
+        addTo(property: any, value: any): void;
+        subtractFrom(property: any, value: any): void;
+        getMetadata(): any;
+        invalidate(options: any): void;
+        /**
+         * This is intended to acquire a lock on a record from the whole cluster.
+         */
+        lock(): void;
+        /**
+         * Store the provided record data into the current resource. This is not written
+         * until the corresponding transaction is committed. This will either immediately fail (synchronously) or always
+         * succeed. That doesn't necessarily mean it will "win", another concurrent put could come "after" (monotonically,
+         * even if not chronologically) this one.
+         * @param record
+         * @param options
+         */
+        put(record: any): void;
+        patch(record_update: any): void;
+        _writeUpdate(record_update: any, full_update: boolean, options?: any): void;
+        delete(request?: Query | string): Promise<boolean>;
+        _writeDelete(options?: any): boolean;
+        search(request: Query): AsyncIterable<any>;
+        subscribe(request: SubscriptionRequest): Promise<{
+            listener: (key: any) => any;
+            subscriptions: [];
+            startTime?: number;
+            end(): void;
+            toJSON(): {
+                name: string;
+            };
+            resolveNext: Function;
+            queue: any[];
+            hasDataListeners: boolean;
+            push(message: any): void;
+            send(message: any): void;
+            getNextMessage(): any;
+            on(event_name: any, listener: any): any;
+            [Symbol.asyncIterator](): {
+                queue: import("./IterableEventQueue").IterableEventQueue;
+                push(message: any): void;
+                next(): Promise<unknown> | {
+                    value: any;
+                };
+                return(value: any): {
+                    value: any;
+                    done: boolean;
+                };
+                throw(error: any): {
+                    done: boolean;
+                };
+            };
+            [EventEmitter.captureRejectionSymbol]?<K>(error: Error, event: string | symbol, ...args: any[]): void;
+            addListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            once<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            removeListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            off<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            removeAllListeners(eventName?: string | symbol): any;
+            setMaxListeners(n: number): any;
+            getMaxListeners(): number;
+            listeners<K>(eventName: string | symbol): Function[];
+            rawListeners<K>(eventName: string | symbol): Function[];
+            emit<K>(eventName: string | symbol, ...args: any[]): boolean;
+            listenerCount<K>(eventName: string | symbol, listener?: Function): number;
+            prependListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            prependOnceListener<K>(eventName: string | symbol, listener: (...args: any[]) => void): any;
+            eventNames(): (string | symbol)[];
+        }>;
+        doesExist(): boolean;
+        /**
+         * Publishing a message to a record adds an (observable) entry in the audit log, but does not change
+         * the record at all. This entries should be replicated and trigger subscription listeners.
+         * @param id
+         * @param message
+         * @param options
+         */
+        publish(message: any, options?: any): void;
+        _writePublish(message: any, options?: any): void;
+        validate(record: any, patch?: any): void;
+        getUpdatedTime(): any;
+        wasLoadedFromSource(): boolean | void;
+        post(new_record: any): any;
+        connect(query?: {}): AsyncIterable<any>;
+        getId(): any;
+        getContext(): Context;
+    };
+    /**
+     * Set TTL expiration for records in this table. On retrieval, record timestamps are checked for expiration.
+     * This also informs the scheduling for record eviction.
+     * @param expiration_time Time in seconds until records expire (are stale)
+     * @param eviction_time Time in seconds until records are evicted (removed)
+     */
+    setTTLExpiration(expiration: number | {
+        expiration: number;
+        eviction?: number;
+        scanInterval?: number;
+    }): void;
+    /**
+     * Turn on auditing at runtime
+     */
+    enableAuditing(audit_enabled?: boolean): void;
+    /**
+     * Coerce the id as a string to the correct type for the primary key
+     * @param id
+     * @returns
+     */
+    coerceId(id: string): number | string;
+    dropTable(): Promise<void>;
+    /**
+     * Evicting a record will remove it from a caching table. This is not considered a canonical data change, and it is assumed that retrieving this record from the source will still yield the same record, this is only removing the local copy of the record.
+     */
+    evict(id: any, existing_record: any, existing_version: any): any;
+    operation(operation: any, context: any): any;
+    /**
+     * This is responsible for ordering and select()ing the attributes/properties from returned entries
+     * @param select
+     * @param context
+     * @param filtered
+     * @param ensure_loaded
+     * @param can_skip
+     * @returns
+     */
+    transformToOrderedSelect(entries: any, select: any, sort: any, context: any, read_txn: any, transformToRecord: any): any;
+    /**
+     * This is responsible for select()ing the attributes/properties from returned entries
+     * @param select
+     * @param context
+     * @param filtered
+     * @param ensure_loaded
+     * @param can_skip
+     * @returns
+     */
+    transformEntryForSelect(select: any, context: any, read_txn: any, filtered: any, ensure_loaded?: any, can_skip?: any): (entry: any) => any;
+    addAttributes(attributes_to_add: any): Promise<any>;
+    removeAttributes(names: string[]): Promise<any>;
+    getRecordCount(options: any): {
+        recordCount: number;
+        estimatedRange: number[];
+    } | {
+        recordCount: number;
+        estimatedRange?: undefined;
+    };
+    /**
+     * When attributes have been changed, we update the accessors that are assigned to this table
+     */
+    updatedAttributes(): void;
+    deleteHistory(end_time?: number): Promise<void>;
+    getHistory(start_time?: number, end_time?: number): AsyncGenerator<{
+        id: import("ordered-binary").Key;
+        localTime: any;
+        version: any;
+        type: any;
+        value: any;
+        user: import("ordered-binary").Key;
+    }, void, unknown>;
+    getHistoryOfRecord(id: any): Promise<any[]>;
+    cleanup(): void;
+    transactions: import("./DatabaseTransaction").Transaction[] & {
+        timestamp: number;
+    };
+    get(identifier: Id, context?: Context): Promise<Resource>;
+    get(query: Query, context?: Context): Promise<AsyncIterable<object>>;
+    put: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    patch: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    delete(identifier: Id, context?: Context): Promise<boolean>;
+    delete(request: Context, context?: Context): Promise<object>;
+    getNewId(): `${string}-${string}-${string}-${string}-${string}`;
+    create(id_prefix: Id, record: any, context: Context): Promise<Id>;
+    create(record: any, context: Context): Promise<Id>;
+    invalidate: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    post: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    connect: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    subscribe(request: SubscriptionRequest): Promise<AsyncIterable<{
+        id: any;
+        operation: string;
+        value: object;
+    }>>;
+    publish: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    search: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    query: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    copy: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    move: {
+        (id_or_query: string | Id, data_or_context?: any, context?: Context): any;
+        reliesOnPrototype: boolean;
+    };
+    isCollection(resource: any): any;
+    parseQuery(search: any): any;
+    parsePath(path: any, context: any, query: any): any;
+};
+export declare function setServerUtilities(utilities: any): void;
+/**
+ * Coerce a string to the type defined by the attribute
+ * @param value
+ * @param attribute
+ * @returns
+ */
+export declare function coerceType(value: any, attribute: any): any;
+export declare function updateResource(resource: any, entry: any): void;