harperdb 4.4.0-alpha.1 → 4.4.0-beta.2

package/package.json

@@ -1,135 +1,132 @@
 {
-  "name": "harperdb",
-  "version": "4.4.0-alpha.1",
-  "description": "HarperDB is a distributed database, caching service, streaming broker, and application development platform focused on performance and ease of use.",
-  "keywords": [
-    "database",
-    "nosql",
-    "api",
-    "distributed",
-    "broker",
-    "mqtt",
-    "real-time",
-    "enterprise",
-    "Fastify",
-    "NATS",
-    "HarperDB",
-    "Harper",
-    "clustering",
-    "replication",
-    "REST",
-    "WebSockets",
-    "decentralized",
-    "server-sent events",
-    "document store"
-  ],
-  "main": "harperdb.js",
-  "bin": {
-    "harperdb": "./bin/harperdb.js"
-  },
-  "engines": {
-    "minimum-node": "16.0.0",
-    "go-lang": "1.21.7",
-    "nats-server": "2.10.11"
-  },
-  "exports": {
-    ".": "./index.js"
-  },
-  "homepage": "https://www.harperdb.io/",
-  "bugs": "support@harperdb.io",
-  "author": {
-    "name": "HarperDB",
-    "email": "support@harperdb.io"
-  },
-  "license": "SEE LICENSE IN LICENSE",
-  "dependencies": {
-    "@aws-sdk/client-s3": "3.635.0",
-    "@aws-sdk/lib-storage": "3.635.0",
-    "@endo/static-module-record": "^1.0.4",
-    "@fastify/autoload": "5.10.0",
-    "@fastify/compress": "~6.5.0",
-    "@fastify/cors": "~9.0.1",
-    "@fastify/static": "~7.0.4",
-    "@turf/area": "6.5.0",
-    "@turf/boolean-contains": "6.5.0",
-    "@turf/boolean-disjoint": "6.5.0",
-    "@turf/boolean-equal": "6.5.0",
-    "@turf/circle": "6.5.0",
-    "@turf/difference": "6.5.0",
-    "@turf/distance": "6.5.0",
-    "@turf/helpers": "6.5.0",
-    "@turf/length": "6.5.0",
-    "alasql": "4.1.10",
-    "cbor-x": "1.6.0",
-    "chalk": "4.1.2",
-    "cli-progress": "3.12.0",
-    "clone": "2.1.2",
-    "fast-glob": "3.3.2",
-    "fastify": "~4.28.1",
-    "fastify-plugin": "~4.5.1",
-    "fs-extra": "11.2.0",
-    "graphql": "^16.9.0",
-    "gunzip-maybe": "1.4.2",
-    "human-readable-ids": "1.0.4",
-    "inquirer": "8.2.6",
-    "is-number": "7.0.0",
-    "joi": "17.12.2",
-    "json-bigint-fixes": "1.1.0",
-    "json2csv": "5.0.7",
-    "jsonata": "1.8.7",
-    "jsonwebtoken": "9.0.2",
-    "lmdb": "3.0.14",
-    "lodash": "4.17.21",
-    "mathjs": "11.12.0",
-    "minimist": "1.2.8",
-    "moment": "2.30.1",
-    "mqtt-packet": "~9.0.0",
-    "msgpackr": "1.11.0",
-    "nats": "2.19.0",
-    "needle": "3.3.1",
-    "node-forge": "^1.3.1",
-    "node-stream-zip": "1.15.0",
-    "node-unix-socket": "0.2.5",
-    "normalize-path": "^3.0.0",
-    "ora": "5.4.1",
-    "ordered-binary": "1.5.1",
-    "papaparse": "5.4.1",
-    "passport": "0.6.0",
-    "passport-http": "0.3.0",
-    "passport-local": "1.0.0",
-    "pino": "8.16.0",
-    "pm2": "5.4.1",
-    "prompt": "1.3.0",
-    "properties-reader": "2.3.0",
-    "recursive-iterator": "3.3.0",
-    "semver": "7.5.4",
-    "send": "^0.18.0",
-    "serve-static": "1.15.0",
-    "ses": "1.1.0",
-    "stream-chain": "2.2.5",
-    "stream-json": "1.8.0",
-    "systeminformation": "5.23.5",
-    "tar-fs": "3.0.6",
-    "ulidx": "0.5.0",
-    "uuid": "10.0.0",
-    "validate.js": "0.13.1",
-    "ws": "8.18.0",
-    "yaml": "2.5.0"
-  },
-  "overrides": {
-    "eslint-plugin-radar": {
-      "eslint": "8.22.0"
-    },
-    "alasql": {
-      "xlsx": "0.18.5"
-    }
-  },
-  "optionalDependencies": {
-    "bufferutil": "^4.0.7",
-    "segfault-handler": "^1.3.0",
-    "utf-8-validate": "^5.0.10"
-  },
-  "scripts": {
-    "postinstall": "node ./launchServiceScripts/launchInstallNATSServer.js"
-  }
+	"name": "harperdb",
+	"version": "4.4.0-beta.2",
+	"description": "HarperDB is a distributed database, caching service, streaming broker, and application development platform focused on performance and ease of use.",
+	"keywords": [
+		"database",
+		"nosql",
+		"api",
+		"distributed",
+		"broker",
+		"mqtt",
+		"real-time",
+		"enterprise",
+		"Fastify",
+		"NATS",
+		"HarperDB",
+		"Harper",
+		"clustering",
+		"replication",
+		"REST",
+		"WebSockets",
+		"decentralized",
+		"server-sent events",
+		"document store"
+	],
+	"main": "harperdb.js",
+	"bin": {
+		"harperdb": "./bin/harperdb.js"
+	},
+	"engines": {
+		"minimum-node": "16.0.0",
+		"go-lang": "1.21.7",
+		"nats-server": "2.10.11"
+	},
+	"exports": {
+		".": "./index.js"
+	},
+	"homepage": "https://www.harperdb.io/",
+	"bugs": "support@harperdb.io",
+	"author": {
+		"name": "HarperDB",
+		"email": "support@harperdb.io"
+	},
+	"license": "SEE LICENSE IN LICENSE",
+	"dependencies": {
+		"@aws-sdk/client-s3": "3.635.0",
+		"@aws-sdk/lib-storage": "3.635.0",
+		"@endo/static-module-record": "^1.0.4",
+		"@fastify/autoload": "5.10.0",
+		"@fastify/compress": "~6.5.0",
+		"@fastify/cors": "~9.0.1",
+		"@fastify/static": "~7.0.4",
+		"@turf/area": "6.5.0",
+		"@turf/boolean-contains": "6.5.0",
+		"@turf/boolean-disjoint": "6.5.0",
+		"@turf/boolean-equal": "6.5.0",
+		"@turf/circle": "6.5.0",
+		"@turf/difference": "6.5.0",
+		"@turf/distance": "6.5.0",
+		"@turf/helpers": "6.5.0",
+		"@turf/length": "6.5.0",
+		"alasql": "4.1.10",
+		"cbor-x": "1.6.0",
+		"chalk": "4.1.2",
+		"cli-progress": "3.12.0",
+		"clone": "2.1.2",
+		"fast-glob": "3.3.2",
+		"fastify": "~4.28.1",
+		"fastify-plugin": "~4.5.1",
+		"fs-extra": "11.2.0",
+		"graphql": "^16.9.0",
+		"gunzip-maybe": "1.4.2",
+		"human-readable-ids": "1.0.4",
+		"inquirer": "8.2.6",
+		"is-number": "7.0.0",
+		"joi": "17.12.2",
+		"json-bigint-fixes": "1.1.0",
+		"json2csv": "5.0.7",
+		"jsonata": "1.8.7",
+		"jsonwebtoken": "9.0.2",
+		"lmdb": "3.1.3",
+		"lodash": "4.17.21",
+		"mathjs": "11.12.0",
+		"minimist": "1.2.8",
+		"moment": "2.30.1",
+		"mqtt-packet": "~9.0.0",
+		"msgpackr": "1.11.0",
+		"nats": "2.19.0",
+		"needle": "3.3.1",
+		"node-forge": "^1.3.1",
+		"node-stream-zip": "1.15.0",
+		"node-unix-socket": "0.2.5",
+		"normalize-path": "^3.0.0",
+		"ora": "5.4.1",
+		"ordered-binary": "1.5.2",
+		"papaparse": "5.4.1",
+		"passport": "0.6.0",
+		"passport-http": "0.3.0",
+		"passport-local": "1.0.0",
+		"pino": "8.16.0",
+		"pm2": "5.4.1",
+		"prompt": "1.3.0",
+		"properties-reader": "2.3.0",
+		"recursive-iterator": "3.3.0",
+		"semver": "7.5.4",
+		"send": "^1.1.0",
+		"serve-static": "2.1.0",
+		"ses": "1.1.0",
+		"stream-chain": "2.2.5",
+		"stream-json": "1.8.0",
+		"systeminformation": "5.23.5",
+		"tar-fs": "3.0.6",
+		"ulidx": "0.5.0",
+		"uuid": "10.0.0",
+		"validate.js": "0.13.1",
+		"ws": "8.18.0",
+		"yaml": "2.5.0"
+	},
+	"overrides": {
+		"alasql": {
+			"xlsx": "0.18.5"
+		}
+	},
+	"optionalDependencies": {
+		"bufferutil": "^4.0.7",
+		"segfault-handler": "^1.3.0",
+		"utf-8-validate": "^5.0.10"
+	},
+	"scripts": {
+		"postinstall": "node ./launchServiceScripts/launchInstallNATSServer.js"
+	}
 }

package/resources/ErrorResource.d.ts

@@ -0,0 +1,26 @@
+import { Resource } from './Resource';
+import { Context } from './ResourceInterface';
+/**
+ * ErrorResource is a Resource that throws an error on any request, communicating to the client when attempts are made
+ * to access endpoints/resources that had an internal error in their configuration or setup. This helps ensure that
+ * if there is a problem with a resource, it is immediately apparent and can be fixed.
+ */
+export declare class ErrorResource implements Resource {
+    error: Error;
+    constructor(error: Error);
+    isError: boolean;
+    allowRead(): never;
+    allowUpdate(): never;
+    allowCreate(): never;
+    allowDelete(): never;
+    getId(): never;
+    getContext(): Context;
+    get(): never;
+    post(): never;
+    put(): never;
+    delete(): never;
+    connect(): never;
+    getResource(): this;
+    publish(): never;
+    subscribe(): never;
+}

package/resources/IterableEventQueue.d.ts

@@ -3,10 +3,16 @@ export declare class IterableEventQueue extends EventEmitter {
     resolveNext: Function;
     queue: any[];
     hasDataListeners: boolean;
+    drainCloseListener: boolean;
+    currentDrainResolver: Function;
     [Symbol.asyncIterator](): EventQueueIterator;
     push(message: any): void;
     send(message: any): void;
     getNextMessage(): any;
+    /**
+     * Wait for the queue to be drained, resolving to true to continue or false if the queue was closed before draining.
+     */
+    waitForDrain(): Promise<boolean>;
     on(event_name: any, listener: any): this;
 }
 declare class EventQueueIterator {

package/resources/Resource.d.ts

@@ -135,4 +135,10 @@ export declare class Resource implements ResourceInterface {
     getContext(): Context;
 }
 export declare function snake_case(camelCase: string): string;
+/**
+ * An array for ids that toString's back to slash-delimited string
+ */
+export declare class MultiPartId extends Array {
+    toString(): string;
+}
 export declare function transformForSelect(select: any, resource: any): (object: any) => any;

package/resources/ResourceInterface.d.ts

@@ -37,7 +37,7 @@ export interface Context {
     /**	 An array of nodes to replicate to */
     replicateTo?: string[];
 }
-interface DirectCondition {
+export interface DirectCondition {
     attribute: string;
     comparator?: string;
     value: any;

package/resources/Table.d.ts

@@ -14,6 +14,8 @@ type Attribute = {
     expiresAt?: boolean;
     isPrimaryKey?: boolean;
 };
+export declare const INVALIDATED = 1;
+export declare const EVICTED = 8;
 export interface Table {
     primaryStore: Database;
     auditStore: Database;
@@ -23,13 +25,12 @@ export interface Table {
     databaseName: string;
     attributes: any[];
     primaryKey: string;
+    splitSegments?: boolean;
     replicate?: boolean;
     subscriptions: Map<any, Function[]>;
     expirationMS: number;
     indexingOperations?: Promise<void>;
-    sources: {
-        new (): ResourceInterface;
-    }[];
+    sources: (new () => ResourceInterface)[];
     Transaction: ReturnType<typeof makeTable>;
 }
 /**
@@ -116,9 +117,12 @@ export declare function makeTable(options: any): {
             resolveNext: Function;
             queue: any[];
             hasDataListeners: boolean;
+            drainCloseListener: boolean;
+            currentDrainResolver: Function;
             push(message: any): void;
             send(message: any): void;
             getNextMessage(): any;
+            waitForDrain(): Promise<boolean>;
             on(event_name: any, listener: any): any;
             [Symbol.asyncIterator](): {
                 queue: import("./IterableEventQueue").IterableEventQueue;
@@ -181,6 +185,7 @@ export declare function makeTable(options: any): {
     attributes: any;
     replicate: any;
     sealed: any;
+    splitSegments: any;
     createdTimeProperty: Attribute;
     updatedTimeProperty: Attribute;
     propertyResolvers: any;
@@ -284,9 +289,12 @@ export declare function makeTable(options: any): {
             resolveNext: Function;
             queue: any[];
             hasDataListeners: boolean;
+            drainCloseListener: boolean;
+            currentDrainResolver: Function;
             push(message: any): void;
             send(message: any): void;
             getNextMessage(): any;
+            waitForDrain(): Promise<boolean>;
             on(event_name: any, listener: any): any;
             [Symbol.asyncIterator](): {
                 queue: import("./IterableEventQueue").IterableEventQueue;
@@ -413,9 +421,12 @@ export declare function makeTable(options: any): {
             resolveNext: Function;
             queue: any[];
             hasDataListeners: boolean;
+            drainCloseListener: boolean;
+            currentDrainResolver: Function;
             push(message: any): void;
             send(message: any): void;
             getNextMessage(): any;
+            waitForDrain(): Promise<boolean>;
             on(event_name: any, listener: any): any;
             [Symbol.asyncIterator](): {
                 queue: import("./IterableEventQueue").IterableEventQueue;

package/resources/auditStore.d.ts

@@ -33,18 +33,18 @@ export declare const HAS_EXPIRATION_EXTENDED_TYPE = 4096;
  * @param previous_residency_id
  */
 export declare function createAuditEntry(txn_time: any, table_id: any, record_id: any, previous_local_time: any, node_id: any, username: any, type: any, encoded_record: any, extended_type: any, residency_id: any, previous_residency_id: any, expires_at: any, originating_operation?: string): Buffer;
-export declare function readAuditEntry(buffer: any): {
+export declare function readAuditEntry(buffer: Uint8Array, start?: number, end?: any): {
     type: any;
     tableId: any;
     nodeId: any;
     readonly recordId: import("ordered-binary").Key;
-    getBinaryRecordId(): any;
+    getBinaryRecordId(): Uint8Array;
     version: any;
     previousLocalTime: any;
     readonly user: import("ordered-binary").Key;
-    encoded: any;
+    readonly encoded: Uint8Array;
     getValue(store: any, full_record?: any, audit_time?: any): any;
-    getBinaryValue(): any;
+    getBinaryValue(): Uint8Array;
     extendedType: any;
     residencyId: any;
     previousResidencyId: any;
@@ -59,7 +59,7 @@ export declare function readAuditEntry(buffer: any): {
     version?: undefined;
     previousLocalTime?: undefined;
     readonly user?: undefined;
-    encoded?: undefined;
+    readonly encoded?: undefined;
     getValue?: undefined;
     getBinaryValue?: undefined;
     extendedType?: undefined;

package/resources/crdt.d.ts

@@ -7,7 +7,7 @@ export declare namespace add {
  * @param update
  * @param newer_update
  */
-export declare function rebuildUpdateBefore(update: any, newer_update: any): any;
+export declare function rebuildUpdateBefore(update: any, newer_update: any, full_update?: boolean): any;
 export declare function applyReverse(record: any, update: any): void;
 /**
  * Reconstruct the record state at a given timestamp by going back through the audit history and reversing any changes

package/resources/databases.d.ts

@@ -39,6 +39,7 @@ interface TableDefinition {
     scanInterval?: number;
     audit?: boolean;
     sealed?: boolean;
+    splitSegments?: boolean;
     replicate?: boolean;
     trackDeletes?: boolean;
     attributes: any[];
@@ -70,9 +71,10 @@ export declare function dropDatabase(database_name: any): Promise<void>;
  * @param attributes
  * @param audit
  * @param sealed
+ * @param splitSegments
  * @param replicate
  */
-export declare function table({ table: table_name, database: database_name, expiration, eviction, scanInterval: scan_interval, attributes, audit, sealed, replicate, trackDeletes: track_deletes, schemaDefined: schema_defined, origin, }: TableDefinition): {
+export declare function table(table_definition: TableDefinition): {
     new (identifier: import("./ResourceInterface").Id, source: any): {
         ensureLoaded(): any;
         get(query?: import("./ResourceInterface").Query | string): Promise<object | void> | object | void;
@@ -105,9 +107,12 @@ export declare function table({ table: table_name, database: database_name, expi
             resolveNext: Function;
             queue: any[];
             hasDataListeners: boolean;
+            drainCloseListener: boolean;
+            currentDrainResolver: Function;
             push(message: any): void;
             send(message: any): void;
             getNextMessage(): any;
+            waitForDrain(): Promise<boolean>;
             on(event_name: any, listener: any): any;
             [Symbol.asyncIterator](): {
                 queue: import("./IterableEventQueue").IterableEventQueue;
@@ -163,6 +168,7 @@ export declare function table({ table: table_name, database: database_name, expi
     attributes: any;
     replicate: any;
     sealed: any;
+    splitSegments: any;
     createdTimeProperty: {
         name: string;
         type: string;
@@ -218,9 +224,12 @@ export declare function table({ table: table_name, database: database_name, expi
             resolveNext: Function;
             queue: any[];
             hasDataListeners: boolean;
+            drainCloseListener: boolean;
+            currentDrainResolver: Function;
             push(message: any): void;
             send(message: any): void;
             getNextMessage(): any;
+            waitForDrain(): Promise<boolean>;
             on(event_name: any, listener: any): any;
             [Symbol.asyncIterator](): {
                 queue: import("./IterableEventQueue").IterableEventQueue;
@@ -294,9 +303,12 @@ export declare function table({ table: table_name, database: database_name, expi
             resolveNext: Function;
             queue: any[];
             hasDataListeners: boolean;
+            drainCloseListener: boolean;
+            currentDrainResolver: Function;
             push(message: any): void;
             send(message: any): void;
             getNextMessage(): any;
+            waitForDrain(): Promise<boolean>;
             on(event_name: any, listener: any): any;
             [Symbol.asyncIterator](): {
                 queue: import("./IterableEventQueue").IterableEventQueue;

package/resources/search.d.ts

@@ -1,3 +1,4 @@
+import { DirectCondition, Id } from './ResourceInterface';
 export declare const COERCIBLE_OPERATORS: {
     lt: boolean;
     le: boolean;
@@ -7,7 +8,19 @@ export declare const COERCIBLE_OPERATORS: {
     eq: boolean;
 };
 export declare function executeConditions(conditions: any, operator: any, table: any, txn: any, request: any, context: any, transformToEntries: any, filtered: any): any;
-export declare function searchByIndex(search_condition: any, transaction: any, reverse: any, Table: any, allow_full_scan?: any, filtered?: any): any;
+/**
+ * Search for records or keys, based on the search condition, using an index if available
+ * @param search_condition
+ * @param transaction
+ * @param reverse
+ * @param Table
+ * @param allow_full_scan
+ * @param filtered
+ */
+export declare function searchByIndex(search_condition: DirectCondition, transaction: any, reverse: boolean, Table: any, allow_full_scan?: boolean, filtered?: boolean): AsyncIterable<Id | {
+    key: Id;
+    value: any;
+}>;
 export declare function findAttribute(attributes: any, attribute_name: any): any;
 /**
  * Create a filter based on the search condition that can be used to test each supplied record.
@@ -28,5 +41,5 @@ export declare class SimpleURLQuery {
     url: string;
     constructor(url: string);
     get(): void;
-    [Symbol.iterator](): IterableIterator<any>;
+    [Symbol.iterator](): ArrayIterator<any>;
 }

package/resources/tracked.d.ts

@@ -24,7 +24,7 @@ export declare function collapseData(target: any): any;
  * @param target
  * @returns
  */
-export declare function deepFreeze(target: any, changes?: any): any;
+export declare function updateAndFreeze(target: any, changes?: any): any;
 /**
  * Determine if any changes have been made to this tracked object
  * @param target