@harperfast/harper 5.0.0-alpha.10

package/dist/resources/RecordEncoder.js

@@ -0,0 +1,700 @@
+"use strict";
+/**
+ * This module is responsible for handling metadata encoding and decoding in database records, which is
+ * used for local timestamps (that lmdb-js can assign during a transaction for guaranteed monotonic
+ * assignment across threads) and can be used for storing residency information as well. This
+ * patches the primary store to properly get the metadata and assign it to the entries.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecordEncoder = exports.lastMetadata = exports.entryMap = exports.HAS_ADDITIONAL_AUDIT_REFS = exports.HAS_STRUCTURE_UPDATE = exports.PENDING_LOCAL_TIME = exports.HAS_NODE_ID = exports.HAS_RESIDENCY_ID = exports.HAS_EXPIRATION = exports.TIMESTAMP_RECORD_PREVIOUS = exports.TIMESTAMP_ASSIGN_PREVIOUS = exports.TIMESTAMP_ASSIGN_LAST = exports.TIMESTAMP_ASSIGN_NEW = exports.NO_TIMESTAMP = exports.ENTRY = exports.METADATA = exports.LOCAL_TIMESTAMP = exports.NEW_TIMESTAMP_PLACEHOLDER = exports.PREVIOUS_TIMESTAMP_PLACEHOLDER = exports.LAST_TIMESTAMP_PLACEHOLDER = exports.TIMESTAMP_PLACEHOLDER = void 0;
+exports.handleLocalTimeForGets = handleLocalTimeForGets;
+exports.recordUpdater = recordUpdater;
+exports.setAdditionalAuditRefs = setAdditionalAuditRefs;
+exports.removeEntry = removeEntry;
+const msgpackr_1 = require("msgpackr");
+const auditStore_ts_1 = require("./auditStore.js");
+const harperLogger = __importStar(require("../utility/logging/harper_logger.js"));
+require("./blob.js");
+const blob_ts_1 = require("./blob.js");
+const write_ts_1 = require("./analytics/write.js");
+const rocksdb_js_1 = require("@harperfast/rocksdb-js");
+const when_ts_1 = require("../utility/when.js");
+// these are matched by lmdb-js for timestamp replacement. the first byte here is used to xor with the first byte of the date as a double so that it ends up less than 32 for easier identification (otherwise dates start with 66)
+exports.TIMESTAMP_PLACEHOLDER = new Uint8Array([1, 1, 1, 1, 4, 0x40, 0, 0]);
+// the first byte here indicates that we use the last timestamp
+exports.LAST_TIMESTAMP_PLACEHOLDER = new Uint8Array([1, 1, 1, 1, 1, 0, 0, 0]);
+exports.PREVIOUS_TIMESTAMP_PLACEHOLDER = new Uint8Array([1, 1, 1, 1, 3, 0x40, 0, 0]);
+exports.NEW_TIMESTAMP_PLACEHOLDER = new Uint8Array([1, 1, 1, 1, 0, 0x40, 0, 0]);
+exports.LOCAL_TIMESTAMP = Symbol('local-timestamp');
+exports.METADATA = Symbol('metadata');
+exports.ENTRY = Symbol('entry');
+const TIMESTAMP_HOLDER = new Uint8Array(8);
+const TIMESTAMP_VIEW = new DataView(TIMESTAMP_HOLDER.buffer, 0, 8);
+exports.NO_TIMESTAMP = 0;
+exports.TIMESTAMP_ASSIGN_NEW = 0;
+exports.TIMESTAMP_ASSIGN_LAST = 1;
+exports.TIMESTAMP_ASSIGN_PREVIOUS = 3;
+exports.TIMESTAMP_RECORD_PREVIOUS = 4;
+exports.HAS_EXPIRATION = 16;
+exports.HAS_RESIDENCY_ID = 32;
+exports.HAS_NODE_ID = 64;
+exports.PENDING_LOCAL_TIME = 1;
+exports.HAS_STRUCTURE_UPDATE = 0x100;
+exports.HAS_ADDITIONAL_AUDIT_REFS = 0x80;
+const TRACKED_WRITE_TYPES = new Set(['put', 'patch', 'delete', 'message', 'publish']);
+// For now we use this as the private property mechanism for mapping records to entries.
+// WeakMaps are definitely not the fastest form of private properties, but they are the only
+// way to do this with how the objects are frozen for now.
+exports.entryMap = new WeakMap();
+let lastValueEncoding, timestampNextEncoding = 0, metadataInNextEncoding = -1, expiresAtNextEncoding = -1, residencyIdAtNextEncoding = 0, nodeIdAtNextEncoding = -1, additionalAuditRefsNextEncoding;
+// tracking metadata with a singleton works better than trying to alter response of getEntry/get and coordinating that across caching layers
+exports.lastMetadata = null;
+class RecordEncoder extends msgpackr_1.Encoder {
+    structureUpdate;
+    isRocksDB;
+    name;
+    constructor(options) {
+        options.useBigIntExtension = true;
+        /**
+         * The base class for records that provides the read-only methods for accessing
+         * metadata and will be assigned computed property getters. On its own, these instances
+         * are usually frozen, but this can be extended (by the Updatable class) for providing
+         * mutation methods.
+         */
+        class RecordObject {
+            getUpdatedTime() {
+                return exports.entryMap.get(this)?.version;
+            }
+            getExpiresAt() {
+                return exports.entryMap.get(this)?.expiresAt;
+            }
+        }
+        options.structPrototype = RecordObject.prototype;
+        super(options);
+        const superEncode = this.encode;
+        this.encode = function (record, options) {
+            // this handles our custom metadata encoding, prefixing the record with metadata, including the local
+            // timestamp into the audit record, invalidation status and residency information
+            if (timestampNextEncoding || metadataInNextEncoding >= 0) {
+                let valueStart = 0;
+                const timestamp = timestampNextEncoding;
+                if (timestamp) {
+                    valueStart += 8; // make room for local timestamp
+                    timestampNextEncoding = 0;
+                }
+                let metadata = metadataInNextEncoding;
+                const expiresAt = expiresAtNextEncoding;
+                const residencyId = residencyIdAtNextEncoding;
+                const nodeId = nodeIdAtNextEncoding;
+                const additionalAuditRefs = additionalAuditRefsNextEncoding;
+                if (metadata >= 0) {
+                    valueStart += 4; // make room for metadata bytes
+                    metadataInNextEncoding = -1; // reset indicator to mean no metadata
+                    if (expiresAt >= 0) {
+                        valueStart += 8; // make room for expiration timestamp
+                        expiresAtNextEncoding = -1; // reset indicator to mean no expiration
+                        if (!(metadata & exports.HAS_EXPIRATION)) {
+                            throw new Error('Expiration included, but not in metadata flags');
+                        }
+                    }
+                    if (residencyId) {
+                        valueStart += 4; // make room for residency id
+                        residencyIdAtNextEncoding = 0; // reset indicator to mean no residency id
+                        if (!(metadata & exports.HAS_RESIDENCY_ID)) {
+                            throw new Error('Residency id included, but not in metadata flags');
+                        }
+                    }
+                    if (nodeId >= 0) {
+                        valueStart += 4; // make room for node id
+                        nodeIdAtNextEncoding = -1; // reset indicator to mean no node id
+                        if (!(metadata & exports.HAS_NODE_ID)) {
+                            throw new Error('Node id included, but not in metadata flags');
+                        }
+                    }
+                    if (additionalAuditRefs && additionalAuditRefs.length > 0) {
+                        valueStart += 1 + additionalAuditRefs.length * 12; // 1 byte for count + 8 bytes version + 4 bytes nodeId per ref
+                        additionalAuditRefsNextEncoding = undefined;
+                    }
+                }
+                const encoded = superEncode.call(this, record, options | 2048 | valueStart); // encode with 8 bytes reserved space for txnId
+                lastValueEncoding = encoded.subarray((encoded.start || 0) + valueStart, encoded.end);
+                let position = encoded.start || 0;
+                const dataView = encoded.dataView || (encoded.dataView = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength));
+                if (timestamp) {
+                    if (this.isRocksDB) {
+                        // rocksdb, just store the version directly as the timestamp
+                        dataView.setFloat64(position, timestamp);
+                    }
+                    else {
+                        // we apply the special instruction bytes that tell lmdb-js how to assign the timestamp
+                        exports.TIMESTAMP_PLACEHOLDER[4] = timestamp;
+                        exports.TIMESTAMP_PLACEHOLDER[5] = timestamp >> 8;
+                        encoded.set(exports.TIMESTAMP_PLACEHOLDER, position);
+                    }
+                    position += 8;
+                }
+                if (blob_ts_1.blobsWereEncoded)
+                    metadata |= auditStore_ts_1.HAS_BLOBS;
+                if (additionalAuditRefs && additionalAuditRefs.length > 0)
+                    metadata |= exports.HAS_ADDITIONAL_AUDIT_REFS;
+                if (metadata >= 0) {
+                    dataView.setUint32(position, metadata | (auditStore_ts_1.ACTION_32_BIT << 24)); // use the extended action byte
+                    position += 4;
+                    if (expiresAt >= 0) {
+                        dataView.setFloat64(position, expiresAt);
+                        position += 8;
+                    }
+                    if (residencyId) {
+                        dataView.setUint32(position, residencyId);
+                        position += 4;
+                    }
+                    if (nodeId >= 0) {
+                        dataView.setUint32(position, nodeId);
+                        position += 4;
+                    }
+                    if (additionalAuditRefs && additionalAuditRefs.length > 0) {
+                        encoded[position++] = additionalAuditRefs.length;
+                        for (const ref of additionalAuditRefs) {
+                            dataView.setFloat64(position, ref.version);
+                            position += 8;
+                            dataView.setUint32(position, ref.nodeId);
+                            position += 4;
+                        }
+                    }
+                }
+                return encoded;
+            }
+            else {
+                lastValueEncoding = superEncode.call(this, record, options);
+                return lastValueEncoding;
+            }
+        };
+        const superSaveStructures = this.saveStructures;
+        const superGetStructures = this.getStructures;
+        this.saveStructures = function (structures, isCompatible) {
+            if (this.isRocksDB) {
+                return this.rootStore.transactionSync((txn) => {
+                    const sharedStructuresKey = [Symbol.for('structures'), this.name];
+                    const existingStructuresBuffer = txn.getBinarySync(sharedStructuresKey);
+                    const existingStructures = existingStructuresBuffer ? this.decode(existingStructuresBuffer) : undefined;
+                    if (typeof isCompatible == 'function') {
+                        if (!isCompatible(existingStructures)) {
+                            return false;
+                        }
+                    }
+                    else if (existingStructures && existingStructures.length !== isCompatible) {
+                        return false;
+                    }
+                    txn.putSync(sharedStructuresKey, structures);
+                    this.structureUpdate = structures;
+                });
+            }
+            else {
+                const result = superSaveStructures.call(this, structures, isCompatible);
+                this.structureUpdate = structures;
+                return result;
+            }
+        };
+        this.getStructures = function () {
+            if (this.isRocksDB) {
+                const sharedStructuresKey = [Symbol.for('structures'), this.name];
+                const buffer = this.rootStore.getBinarySync(sharedStructuresKey);
+                return buffer ? this.decode(buffer) : undefined;
+            }
+            else {
+                return superGetStructures.call(this);
+            }
+        };
+    }
+    decode(buffer, options) {
+        exports.lastMetadata = null;
+        const start = options?.start || 0;
+        const end = options > -1 ? options : options?.end || buffer.length;
+        let nextByte = buffer[start];
+        let metadataFlags = 0;
+        try {
+            if ((this.isRocksDB && nextByte === 66) || (nextByte < 32 && end > 2)) {
+                // record with metadata
+                // this means that the record starts with a local timestamp (that was assigned by lmdb-js).
+                // we copy it so we can decode it as float-64; we need to do it first because if structural data
+                // is loaded during decoding the buffer can actually mutate
+                let position = start;
+                let localTime;
+                if (this.isRocksDB) {
+                    buffer.copy(TIMESTAMP_HOLDER, 0, position);
+                    position += 8;
+                    localTime = TIMESTAMP_VIEW.getFloat64(0);
+                    nextByte = buffer[position];
+                }
+                else if (nextByte === 2) {
+                    if (buffer.copy) {
+                        buffer.copy(TIMESTAMP_HOLDER, 0, position);
+                        position += 8;
+                    }
+                    else {
+                        for (let i = 0; i < 8; i++)
+                            TIMESTAMP_HOLDER[i] = buffer[position++];
+                    }
+                    localTime = getTimestamp();
+                    nextByte = buffer[position];
+                }
+                let expiresAt, residencyId, nodeId, additionalAuditRefs;
+                if (nextByte < 32) {
+                    if (nextByte === auditStore_ts_1.ACTION_32_BIT) {
+                        const dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength));
+                        metadataFlags = dataView.getUint32(position);
+                        position += 4;
+                    }
+                    else {
+                        metadataFlags = nextByte | (buffer[position + 1] << 5);
+                        position += 2;
+                    }
+                    if (metadataFlags & exports.HAS_EXPIRATION) {
+                        const dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength));
+                        expiresAt = dataView.getFloat64(position);
+                        position += 8;
+                    }
+                    if (metadataFlags & exports.HAS_RESIDENCY_ID) {
+                        // we need to read the residency id
+                        const dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength));
+                        residencyId = dataView.getUint32(position);
+                        position += 4;
+                    }
+                    if (metadataFlags & exports.HAS_NODE_ID) {
+                        // we need to read the node id
+                        const dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength));
+                        nodeId = dataView.getUint32(position);
+                        position += 4;
+                    }
+                    if (metadataFlags & exports.HAS_ADDITIONAL_AUDIT_REFS) {
+                        // we need to read the additional audit refs
+                        const dataView = buffer.dataView || (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength));
+                        const count = buffer[position++];
+                        additionalAuditRefs = [];
+                        for (let i = 0; i < count; i++) {
+                            const version = dataView.getFloat64(position);
+                            position += 8;
+                            const refNodeId = dataView.getUint32(position);
+                            position += 4;
+                            additionalAuditRefs.push({ version, nodeId: refNodeId });
+                        }
+                    }
+                }
+                const value = (0, blob_ts_1.decodeFromDatabase)(() => options?.valueAsBuffer
+                    ? buffer.subarray(position, end)
+                    : super.decode(buffer.subarray(position, end), end - position), this.rootStore);
+                exports.lastMetadata = {
+                    localTime,
+                    version: localTime,
+                    [exports.METADATA]: metadataFlags,
+                    expiresAt,
+                    residencyId,
+                    nodeId,
+                    additionalAuditRefs,
+                    size: end - start,
+                    value,
+                };
+                if (this.isRocksDB)
+                    return exports.lastMetadata;
+                return value;
+            } // else a normal entry
+            return options?.valueAsBuffer ? buffer : (0, blob_ts_1.decodeFromDatabase)(() => super.decode(buffer, options), this.rootStore);
+        }
+        catch (error) {
+            harperLogger.error('Error decoding record', error, 'data: ' + buffer.slice(0, 40).toString('hex'));
+            return null;
+        }
+    }
+}
+exports.RecordEncoder = RecordEncoder;
+function getTimestamp() {
+    TIMESTAMP_HOLDER[0] = TIMESTAMP_HOLDER[0] ^ 0x40; // restore the first byte, we xor to differentiate the first byte from structures
+    return TIMESTAMP_VIEW.getFloat64(0);
+}
+function handleLocalTimeForGets(store, rootStore) {
+    const isRocksDB = store instanceof rocksdb_js_1.RocksDatabase;
+    store.readCount = 0;
+    store.cachePuts = false;
+    store.rootStore = rootStore;
+    store.encoder.rootStore = rootStore;
+    store.encoder.isRocksDB = isRocksDB;
+    store.decoder = store.encoder;
+    const storeGetEntry = store.getEntry;
+    const storeGetSync = store.getSync;
+    const storeGet = store.get;
+    store.getEntry = function (id, options) {
+        store.readCount++;
+        exports.lastMetadata = null;
+        if (isRocksDB) {
+            return (0, when_ts_1.when)(options?.async ? storeGet.call(store, id, options) : storeGetSync.call(store, id, options), (entry) => {
+                if (entry) {
+                    if (entry[exports.METADATA]) {
+                        entry.metadataFlags = entry[exports.METADATA];
+                        return withEntry(entry);
+                    }
+                    else
+                        return { value: entry };
+                }
+                else
+                    return entry;
+            });
+        }
+        else {
+            let entry = storeGetEntry.call(this, id, options);
+            if (exports.lastMetadata) {
+                entry.metadataFlags = exports.lastMetadata[exports.METADATA];
+                entry.localTime = exports.lastMetadata.localTime;
+                entry.residencyId = exports.lastMetadata.residencyId;
+                entry.nodeId = exports.lastMetadata.nodeId;
+                entry.additionalAuditRefs = exports.lastMetadata.additionalAuditRefs;
+                entry.size = exports.lastMetadata.size;
+                if (exports.lastMetadata.expiresAt >= 0) {
+                    entry.expiresAt = exports.lastMetadata.expiresAt;
+                }
+                if (isRocksDB)
+                    entry.version = exports.lastMetadata.localTime;
+                if (entry.value) {
+                    exports.entryMap.set(entry.value, entry); // allow the record to access the entry
+                }
+                entry.key = id;
+            }
+            return entry && withEntry(entry);
+        }
+        // if we have decoded with metadata, we want to pull it out and assign to this entry
+        function withEntry(entry) {
+            if (entry.value) {
+                if (entry.value.constructor === Object) {
+                    // if an object was deserialized as a plain object, give it the right prototype for computed properties to be accessible
+                    const originalValue = entry.value;
+                    entry.value = new this.encoder.structPrototype.constructor();
+                    Object.assign(entry.value, originalValue);
+                }
+                exports.entryMap.set(entry.value, entry); // allow the record to access the entry
+            }
+            entry.key = id;
+            return entry;
+        }
+    };
+    store.getSync = function (id, options) {
+        const entry = store.getEntry(id, options);
+        const value = entry?.value;
+        if (value) {
+            exports.entryMap.set(value, entry);
+        }
+        return value;
+    };
+    store.get = function (id, options) {
+        return (0, when_ts_1.when)(store.getEntry(id, { ...options, async: true }), (entry) => {
+            const value = entry?.value;
+            if (value) {
+                exports.entryMap.set(value, entry);
+            }
+            return value;
+        });
+    };
+    //store.pendingTimestampUpdates = new Map();
+    const storeGetRange = store.getRange;
+    store.getRange = function (options) {
+        const iterable = storeGetRange.call(this, options);
+        if (options.valuesForKey) {
+            return iterable.map((value) => value?.value);
+        }
+        if (options.values === false || options.onlyCount)
+            return iterable;
+        return iterable.map((entry) => {
+            // if we have metadata, move the metadata to the entry
+            if (isRocksDB) {
+                if (entry.value?.[exports.METADATA]) {
+                    entry.metadataFlags = entry.value[exports.METADATA];
+                    Object.assign(entry, entry.value);
+                }
+            }
+            else if (exports.lastMetadata) {
+                entry.metadataFlags = exports.lastMetadata[exports.METADATA];
+                entry.localTime = exports.lastMetadata.localTime;
+                if (isRocksDB)
+                    entry.version = exports.lastMetadata.localTime;
+                entry.residencyId = exports.lastMetadata.residencyId;
+                entry.nodeId = exports.lastMetadata.nodeId;
+                entry.additionalAuditRefs = exports.lastMetadata.additionalAuditRefs;
+                entry.size = exports.lastMetadata.size;
+                if (exports.lastMetadata.expiresAt >= 0)
+                    entry.expiresAt = exports.lastMetadata.expiresAt;
+                exports.lastMetadata = null;
+            }
+            if (entry.value) {
+                if (entry.value.constructor === Object) {
+                    // if an object was deserialized as a plain object, give it the right prototype for computed properties to be accessible
+                    const originalValue = entry.value;
+                    entry.value = new this.encoder.structPrototype.constructor();
+                    for (const key in originalValue)
+                        entry.value[key] = originalValue[key];
+                }
+            }
+            return entry;
+        });
+    };
+    if (!isRocksDB) {
+        // add read transaction tracking
+        const txn = store.useReadTransaction();
+        txn.done();
+        if (!txn.done.isTracked) {
+            const Txn = txn.constructor;
+            const use = txn.use;
+            const done = txn.done;
+            Txn.prototype.use = function () {
+                if (!this.timerTracked) {
+                    this.timerTracked = true;
+                    trackedTxns.push(new WeakRef(this));
+                }
+                use.call(this);
+            };
+            Txn.prototype.done = function () {
+                done.call(this);
+                if (this.isDone) {
+                    for (let i = 0; i < trackedTxns.length; i++) {
+                        const txn = trackedTxns[i].deref();
+                        if (!txn || txn.isDone || txn.isCommitted) {
+                            trackedTxns.splice(i--, 1);
+                        }
+                    }
+                }
+            };
+            Txn.prototype.done.isTracked = true;
+        }
+    }
+    return store;
+}
+const trackedTxns = [];
+setInterval(() => {
+    for (let i = 0; i < trackedTxns.length; i++) {
+        const txn = trackedTxns[i].deref();
+        if (!txn || txn.isDone || txn.isCommitted)
+            trackedTxns.splice(i--, 1);
+        else if (txn.notCurrent) {
+            if (txn.openTimer) {
+                if (txn.openTimer > 3) {
+                    if (txn.openTimer > 60) {
+                        harperLogger.error('Read transaction detected that has been open too long (over 15 minutes), ending transaction', txn);
+                        txn.done();
+                    }
+                    else
+                        harperLogger.error('Read transaction detected that has been open too long (over one minute), make sure read transactions are quickly closed', txn);
+                }
+                txn.openTimer++;
+            }
+            else
+                txn.openTimer = 1;
+        }
+    }
+}, 15000).unref();
+function recordUpdater(store, tableId, auditStore) {
+    return function (id, record, existingEntry, newVersion, assignMetadata = -1, // when positive, this has a set of metadata flags for the record
+    audit, // true -> audit this record. false -> do not. null -> retain any audit timestamp
+    options, type = 'put', resolveRecord, // indicates that we are resolving (from source) record that was previously invalidated
+    auditRecord) {
+        const isRocksDB = store instanceof rocksdb_js_1.RocksDatabase;
+        // determine if and how we apply the local timestamp
+        if (isRocksDB) {
+            // with rocksdb, we simplify to just storing the singular version/timestamp
+            timestampNextEncoding = newVersion;
+        }
+        else if (audit == null)
+            // if not auditing, there is no local timestamp to reference
+            timestampNextEncoding = exports.NO_TIMESTAMP;
+        else if (resolveRecord)
+            // preserve existing timestamp, if possible
+            timestampNextEncoding = existingEntry?.localTime
+                ? exports.TIMESTAMP_RECORD_PREVIOUS | exports.TIMESTAMP_ASSIGN_PREVIOUS
+                : exports.NO_TIMESTAMP;
+        else
+            timestampNextEncoding = audit // for audit, we need it
+                ? existingEntry?.localTime // we already have a timestamp, we need to record the previous one in the audit log
+                    ? exports.TIMESTAMP_RECORD_PREVIOUS | 0x4000
+                    : exports.TIMESTAMP_ASSIGN_NEW | 0x4000 // or just assign a new one
+                : exports.NO_TIMESTAMP;
+        const expiresAt = options?.expiresAt;
+        if (expiresAt >= 0)
+            assignMetadata |= exports.HAS_EXPIRATION;
+        metadataInNextEncoding = assignMetadata;
+        expiresAtNextEncoding = expiresAt;
+        const putOptions = {
+            version: newVersion,
+            instructedWrite: timestampNextEncoding > 0,
+            transaction: options?.transaction,
+        };
+        let ifVersion;
+        let extendedType = 0;
+        try {
+            let previousResidencyId = existingEntry?.residencyId;
+            const residencyId = options?.residencyId; //getResidency(record, previousResidencyId);
+            if (residencyId) {
+                residencyIdAtNextEncoding = residencyId;
+                metadataInNextEncoding |= exports.HAS_RESIDENCY_ID;
+                extendedType |= auditStore_ts_1.HAS_CURRENT_RESIDENCY_ID;
+            }
+            else
+                residencyIdAtNextEncoding = 0;
+            const nodeId = options?.nodeId;
+            if (nodeId >= 0) {
+                nodeIdAtNextEncoding = nodeId;
+                metadataInNextEncoding |= exports.HAS_NODE_ID;
+            }
+            else
+                nodeIdAtNextEncoding = -1;
+            const additionalAuditRefs = options?.additionalAuditRefs;
+            if (additionalAuditRefs && additionalAuditRefs.length > 0) {
+                additionalAuditRefsNextEncoding = additionalAuditRefs;
+                metadataInNextEncoding |= exports.HAS_ADDITIONAL_AUDIT_REFS;
+            }
+            else
+                additionalAuditRefsNextEncoding = undefined;
+            const previousAdditionalAuditRefs = existingEntry?.additionalAuditRefs;
+            if (previousAdditionalAuditRefs && previousAdditionalAuditRefs.length > 0) {
+                extendedType |= auditStore_ts_1.HAS_ADDITIONAL_AUDIT_REFS;
+            }
+            if (previousResidencyId !== residencyId) {
+                extendedType |= auditStore_ts_1.HAS_PREVIOUS_RESIDENCY_ID;
+                if (!previousResidencyId)
+                    previousResidencyId = 0;
+            }
+            if (assignMetadata & exports.HAS_EXPIRATION)
+                extendedType |= auditStore_ts_1.HAS_EXPIRATION_EXTENDED_TYPE; // we need to record the expiration in the audit log
+            if (options?.originatingOperation)
+                extendedType |= auditStore_ts_1.HAS_ORIGINATING_OPERATION;
+            // we use resolveRecord outside of transaction, so must explicitly make it conditional
+            if (resolveRecord)
+                putOptions.ifVersion = ifVersion = existingEntry?.version ?? null;
+            if (existingEntry && existingEntry.value && type !== 'message' && existingEntry.metadataFlags & auditStore_ts_1.HAS_BLOBS) {
+                // delete the old blobs
+                (0, blob_ts_1.deleteBlobsInObject)(existingEntry.value);
+            }
+            let result;
+            if (record !== undefined) {
+                result = (0, blob_ts_1.encodeBlobsWithFilePath)(() => (isRocksDB ? store.putSync(id, record, putOptions) : store.put(id, record, putOptions)), id, store.rootStore);
+                if (blob_ts_1.blobsWereEncoded) {
+                    extendedType |= auditStore_ts_1.HAS_BLOBS;
+                }
+            }
+            if (audit) {
+                const username = typeof options?.user === 'string' ? options.user : options?.user?.username;
+                if (auditRecord) {
+                    (0, blob_ts_1.encodeBlobsWithFilePath)(() => store.encoder.encode(auditRecord), id, store.rootStore);
+                    if (blob_ts_1.blobsWereEncoded) {
+                        extendedType |= auditStore_ts_1.HAS_BLOBS;
+                    }
+                }
+                if (store.encoder?.structureUpdate) {
+                    extendedType |= exports.HAS_STRUCTURE_UPDATE;
+                    store.encoder.structureUpdate = null;
+                }
+                const structureVersion = store.encoder.structures.length + (store.encoder.typedStructs?.length ?? 0);
+                const nodeId = options?.nodeId ?? server.replication?.getThisNodeId(auditStore) ?? 0;
+                const viaNodeId = options?.viaNodeId ?? nodeId;
+                if (resolveRecord && existingEntry?.localTime) {
+                    const replacingId = existingEntry?.localTime;
+                    const replacingEntry = auditStore.get(replacingId, tableId, id);
+                    if (replacingEntry) {
+                        const previousVersion = replacingEntry.previousVersion;
+                        result = auditStore[isRocksDB ? 'putSync' : 'put'](replacingId, {
+                            version: newVersion,
+                            tableId,
+                            recordId: id,
+                            previousVersion,
+                            nodeId,
+                            user: username,
+                            type,
+                            encodedRecord: lastValueEncoding,
+                            extendedType,
+                            residencyId,
+                            previousResidencyId,
+                            expiresAt,
+                            structureVersion,
+                            previousAdditionalAuditRefs,
+                        }, { ifVersion: ifVersion, transaction: options.transaction, nodeId, viaNodeId });
+                        return result;
+                    }
+                }
+                result = auditStore[isRocksDB ? 'putSync' : 'put'](record === undefined ? exports.NEW_TIMESTAMP_PLACEHOLDER : exports.LAST_TIMESTAMP_PLACEHOLDER, {
+                    version: newVersion,
+                    tableId,
+                    recordId: id,
+                    previousVersion: store instanceof rocksdb_js_1.RocksDatabase ? existingEntry?.version : existingEntry?.localTime ? 1 : 0,
+                    nodeId,
+                    user: username,
+                    type,
+                    encodedRecord: lastValueEncoding,
+                    extendedType,
+                    residencyId,
+                    previousResidencyId,
+                    expiresAt,
+                    structureVersion,
+                    originatingOperation: options?.originatingOperation,
+                    previousAdditionalAuditRefs,
+                }, {
+                    // turn off append flag, as we are concerned this may be related to db corruption issues
+                    // append: type !== 'invalidate', // for invalidation, we expect the record to be rewritten, so we don't want to necessarily expect pure sequential writes that create full pages
+                    instructedWrite: true,
+                    ifVersion,
+                    transaction: options.transaction,
+                    nodeId,
+                    viaNodeId,
+                });
+            }
+            if (options?.tableToTrack && TRACKED_WRITE_TYPES.has(type)) {
+                (0, write_ts_1.recordAction)(lastValueEncoding?.length ?? 1, 'db-write', options.tableToTrack, null);
+            }
+            return result;
+        }
+        catch (error) {
+            error.message += ' id: ' + id + ' options: ' + putOptions;
+            throw error;
+        }
+    };
+}
+function setAdditionalAuditRefs(refs) {
+    additionalAuditRefsNextEncoding = refs;
+}
+function removeEntry(store, entry, existingVersion) {
+    if (!entry)
+        return;
+    if (entry.value && entry.metadataFlags & auditStore_ts_1.HAS_BLOBS) {
+        // if it used to have blobs, we need to delete the old blobs
+        (0, blob_ts_1.deleteBlobsInObject)(entry.value);
+    }
+    return store.remove(entry.key, existingVersion);
+}
+//# sourceMappingURL=RecordEncoder.js.map