@nats-io/obj 3.0.0-1

package/lib/objectstore.js

@@ -0,0 +1,730 @@
+"use strict";
+/*
+ * Copyright 2022-2024 The NATS Authors
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ObjectStoreImpl = exports.validateBucket = exports.ObjectStoreStatusImpl = exports.Objm = exports.objectStoreBucketName = exports.objectStoreStreamName = exports.digestType = exports.osPrefix = void 0;
+const internal_1 = require("@nats-io/nats-core/internal");
+const internal_2 = require("@nats-io/jetstream/internal");
+exports.osPrefix = "OBJ_";
+exports.digestType = "SHA-256=";
+function objectStoreStreamName(bucket) {
+    validateBucket(bucket);
+    return `${exports.osPrefix}${bucket}`;
+}
+exports.objectStoreStreamName = objectStoreStreamName;
+function objectStoreBucketName(stream) {
+    if (stream.startsWith(exports.osPrefix)) {
+        return stream.substring(4);
+    }
+    return stream;
+}
+exports.objectStoreBucketName = objectStoreBucketName;
+/**
+ * The entry point to creating and managing new ObjectStore instances.
+ */
+class Objm {
+    js;
+    /**
+     * Creates an instance of the Objm that allows you to create and access ObjectStore.
+     * Note that if the argument is a NatsConnection, default JetStream Options are
+     * used. If you want to set some options, please provide a JetStreamClient instead.
+     * @param nc
+     */
+    constructor(nc) {
+        this.js = (0, internal_2.toJetStreamClient)(nc);
+    }
+    /**
+     * Creates and opens the specified ObjectStore. If the Object already exists, it opens the existing ObjectStore.
+     * @param name
+     * @param opts
+     */
+    create(name, opts = {}) {
+        return this.#maybeCreate(name, opts);
+    }
+    #maybeCreate(name, opts = {}) {
+        if (typeof crypto?.subtle?.digest !== "function") {
+            return Promise.reject(new Error("objectstore: unable to calculate hashes - crypto.subtle.digest with sha256 support is required"));
+        }
+        const { ok, min } = this.js.nc.features.get(internal_1.Feature.JS_OBJECTSTORE);
+        if (!ok) {
+            return Promise.reject(new Error(`objectstore is only supported on servers ${min} or better`));
+        }
+        return ObjectStoreImpl.create(this.js, name, opts);
+    }
+    /**
+     * Returns a list of ObjectStoreInfo for all streams that are identified as
+     * being a ObjectStore (that is having names that have the prefix `OBJ_`)
+     */
+    list() {
+        const filter = (v) => {
+            const slr = v;
+            const streams = slr.streams.filter((v) => {
+                return v.config.name.startsWith(exports.osPrefix);
+            });
+            streams.forEach((si) => {
+                si.config.sealed = si.config.sealed || false;
+                si.config.deny_delete = si.config.deny_delete || false;
+                si.config.deny_purge = si.config.deny_purge || false;
+                si.config.allow_rollup_hdrs = si.config.allow_rollup_hdrs || false;
+            });
+            return streams.map((si) => {
+                return new ObjectStoreStatusImpl(si);
+            });
+        };
+        const subj = `${this.js.prefix}.STREAM.LIST`;
+        return new internal_2.ListerImpl(subj, filter, this.js);
+    }
+}
+exports.Objm = Objm;
+class ObjectStoreStatusImpl {
+    si;
+    backingStore;
+    constructor(si) {
+        this.si = si;
+        this.backingStore = "JetStream";
+    }
+    get bucket() {
+        return objectStoreBucketName(this.si.config.name);
+    }
+    get description() {
+        return this.si.config.description ?? "";
+    }
+    get ttl() {
+        return this.si.config.max_age;
+    }
+    get storage() {
+        return this.si.config.storage;
+    }
+    get replicas() {
+        return this.si.config.num_replicas;
+    }
+    get sealed() {
+        return this.si.config.sealed;
+    }
+    get size() {
+        return this.si.state.bytes;
+    }
+    get streamInfo() {
+        return this.si;
+    }
+    get metadata() {
+        return this.si.config.metadata;
+    }
+    get compression() {
+        if (this.si.config.compression) {
+            return this.si.config.compression !== internal_2.StoreCompression.None;
+        }
+        return false;
+    }
+}
+exports.ObjectStoreStatusImpl = ObjectStoreStatusImpl;
+function validateBucket(name) {
+    const validBucketRe = /^[-\w]+$/;
+    if (!validBucketRe.test(name)) {
+        throw new Error(`invalid bucket name: ${name}`);
+    }
+}
+exports.validateBucket = validateBucket;
+class ObjectInfoImpl {
+    info;
+    hdrs;
+    constructor(oi) {
+        this.info = oi;
+    }
+    get name() {
+        return this.info.name;
+    }
+    get description() {
+        return this.info.description ?? "";
+    }
+    get headers() {
+        if (!this.hdrs) {
+            this.hdrs = internal_1.MsgHdrsImpl.fromRecord(this.info.headers || {});
+        }
+        return this.hdrs;
+    }
+    get options() {
+        return this.info.options;
+    }
+    get bucket() {
+        return this.info.bucket;
+    }
+    get chunks() {
+        return this.info.chunks;
+    }
+    get deleted() {
+        return this.info.deleted ?? false;
+    }
+    get digest() {
+        return this.info.digest;
+    }
+    get mtime() {
+        return this.info.mtime;
+    }
+    get nuid() {
+        return this.info.nuid;
+    }
+    get size() {
+        return this.info.size;
+    }
+    get revision() {
+        return this.info.revision;
+    }
+    get metadata() {
+        return this.info.metadata || {};
+    }
+    isLink() {
+        return (this.info.options?.link !== undefined) &&
+            (this.info.options?.link !== null);
+    }
+}
+function toServerObjectStoreMeta(meta) {
+    const v = {
+        name: meta.name,
+        description: meta.description ?? "",
+        options: meta.options,
+        metadata: meta.metadata,
+    };
+    if (meta.headers) {
+        const mhi = meta.headers;
+        v.headers = mhi.toRecord();
+    }
+    return v;
+}
+function emptyReadableStream() {
+    return new ReadableStream({
+        pull(c) {
+            c.enqueue(new Uint8Array(0));
+            c.close();
+        },
+    });
+}
+class ObjectStoreImpl {
+    jsm;
+    js;
+    stream;
+    name;
+    constructor(name, jsm, js) {
+        this.name = name;
+        this.jsm = jsm;
+        this.js = js;
+    }
+    _checkNotEmpty(name) {
+        if (!name || name.length === 0) {
+            return { name, error: new Error("name cannot be empty") };
+        }
+        return { name };
+    }
+    async info(name) {
+        const info = await this.rawInfo(name);
+        return info ? new ObjectInfoImpl(info) : null;
+    }
+    async list() {
+        const buf = [];
+        const iter = await this.watch({
+            ignoreDeletes: true,
+            includeHistory: true,
+        });
+        for await (const info of iter) {
+            // watch will give a null when it has initialized
+            // for us that is the hint we are done
+            if (info === null) {
+                break;
+            }
+            buf.push(info);
+        }
+        return Promise.resolve(buf);
+    }
+    async rawInfo(name) {
+        const { name: obj, error } = this._checkNotEmpty(name);
+        if (error) {
+            return Promise.reject(error);
+        }
+        const meta = this._metaSubject(obj);
+        try {
+            const m = await this.jsm.streams.getMessage(this.stream, {
+                last_by_subj: meta,
+            });
+            const jc = (0, internal_1.JSONCodec)();
+            const soi = jc.decode(m.data);
+            soi.revision = m.seq;
+            return soi;
+        }
+        catch (err) {
+            if (err.code === "404") {
+                return null;
+            }
+            return Promise.reject(err);
+        }
+    }
+    async _si(opts) {
+        try {
+            return await this.jsm.streams.info(this.stream, opts);
+        }
+        catch (err) {
+            const nerr = err;
+            if (nerr.code === "404") {
+                return null;
+            }
+            return Promise.reject(err);
+        }
+    }
+    async seal() {
+        let info = await this._si();
+        if (info === null) {
+            return Promise.reject(new Error("object store not found"));
+        }
+        info.config.sealed = true;
+        info = await this.jsm.streams.update(this.stream, info.config);
+        return Promise.resolve(new ObjectStoreStatusImpl(info));
+    }
+    async status(opts) {
+        const info = await this._si(opts);
+        if (info === null) {
+            return Promise.reject(new Error("object store not found"));
+        }
+        return Promise.resolve(new ObjectStoreStatusImpl(info));
+    }
+    destroy() {
+        return this.jsm.streams.delete(this.stream);
+    }
+    async _put(meta, rs, opts) {
+        const jsopts = this.js.getOptions();
+        opts = opts || { timeout: jsopts.timeout };
+        opts.timeout = opts.timeout || jsopts.timeout;
+        opts.previousRevision = opts.previousRevision ?? undefined;
+        const { timeout, previousRevision } = opts;
+        const si = this.js.nc.info;
+        const maxPayload = si?.max_payload || 1024;
+        meta = meta || {};
+        meta.options = meta.options || {};
+        let maxChunk = meta.options?.max_chunk_size || 128 * 1024;
+        maxChunk = maxChunk > maxPayload ? maxPayload : maxChunk;
+        meta.options.max_chunk_size = maxChunk;
+        const old = await this.info(meta.name);
+        const { name: n, error } = this._checkNotEmpty(meta.name);
+        if (error) {
+            return Promise.reject(error);
+        }
+        const id = internal_1.nuid.next();
+        const chunkSubj = this._chunkSubject(id);
+        const metaSubj = this._metaSubject(n);
+        const info = Object.assign({
+            bucket: this.name,
+            nuid: id,
+            size: 0,
+            chunks: 0,
+        }, toServerObjectStoreMeta(meta));
+        const d = (0, internal_1.deferred)();
+        const proms = [];
+        const db = new internal_1.DataBuffer();
+        try {
+            const reader = rs ? rs.getReader() : null;
+            const sha = new internal_1.SHA256();
+            while (true) {
+                const { done, value } = reader
+                    ? await reader.read()
+                    : { done: true, value: undefined };
+                if (done) {
+                    // put any partial chunk in
+                    if (db.size() > 0) {
+                        const payload = db.drain();
+                        sha.update(payload);
+                        info.chunks++;
+                        info.size += payload.length;
+                        proms.push(this.js.publish(chunkSubj, payload, { timeout }));
+                    }
+                    // wait for all the chunks to write
+                    await Promise.all(proms);
+                    proms.length = 0;
+                    // prepare the metadata
+                    info.mtime = new Date().toISOString();
+                    const digest = sha.digest("base64");
+                    const pad = digest.length % 3;
+                    const padding = pad > 0 ? "=".repeat(pad) : "";
+                    info.digest = `${exports.digestType}${digest}${padding}`;
+                    info.deleted = false;
+                    // trailing md for the object
+                    const h = (0, internal_1.headers)();
+                    if (typeof previousRevision === "number") {
+                        h.set(internal_2.PubHeaders.ExpectedLastSubjectSequenceHdr, `${previousRevision}`);
+                    }
+                    h.set(internal_2.JsHeaders.RollupHdr, internal_2.JsHeaders.RollupValueSubject);
+                    // try to update the metadata
+                    const pa = await this.js.publish(metaSubj, (0, internal_1.JSONCodec)().encode(info), {
+                        headers: h,
+                        timeout,
+                    });
+                    // update the revision to point to the sequence where we inserted
+                    info.revision = pa.seq;
+                    // if we are here, the new entry is live
+                    if (old) {
+                        try {
+                            await this.jsm.streams.purge(this.stream, {
+                                filter: `$O.${this.name}.C.${old.nuid}`,
+                            });
+                        }
+                        catch (_err) {
+                            // rejecting here, would mean send the wrong signal
+                            // the update succeeded, but cleanup of old chunks failed.
+                        }
+                    }
+                    // resolve the ObjectInfo
+                    d.resolve(new ObjectInfoImpl(info));
+                    // stop
+                    break;
+                }
+                if (value) {
+                    db.fill(value);
+                    while (db.size() > maxChunk) {
+                        info.chunks++;
+                        info.size += maxChunk;
+                        const payload = db.drain(meta.options.max_chunk_size);
+                        sha.update(payload);
+                        proms.push(this.js.publish(chunkSubj, payload, { timeout }));
+                    }
+                }
+            }
+        }
+        catch (err) {
+            // we failed, remove any partials
+            await this.jsm.streams.purge(this.stream, { filter: chunkSubj });
+            d.reject(err);
+        }
+        return d;
+    }
+    putBlob(meta, data, opts) {
+        function readableStreamFrom(data) {
+            return new ReadableStream({
+                pull(controller) {
+                    controller.enqueue(data);
+                    controller.close();
+                },
+            });
+        }
+        if (data === null) {
+            data = new Uint8Array(0);
+        }
+        return this.put(meta, readableStreamFrom(data), opts);
+    }
+    put(meta, rs, opts) {
+        if (meta?.options?.link) {
+            return Promise.reject(new Error("link cannot be set when putting the object in bucket"));
+        }
+        return this._put(meta, rs, opts);
+    }
+    async getBlob(name) {
+        async function fromReadableStream(rs) {
+            const buf = new internal_1.DataBuffer();
+            const reader = rs.getReader();
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    return buf.drain();
+                }
+                if (value && value.length) {
+                    buf.fill(value);
+                }
+            }
+        }
+        const r = await this.get(name);
+        if (r === null) {
+            return Promise.resolve(null);
+        }
+        const vs = await Promise.all([r.error, fromReadableStream(r.data)]);
+        if (vs[0]) {
+            return Promise.reject(vs[0]);
+        }
+        else {
+            return Promise.resolve(vs[1]);
+        }
+    }
+    async get(name) {
+        const info = await this.rawInfo(name);
+        if (info === null) {
+            return Promise.resolve(null);
+        }
+        if (info.deleted) {
+            return Promise.resolve(null);
+        }
+        if (info.options && info.options.link) {
+            const ln = info.options.link.name || "";
+            if (ln === "") {
+                throw new Error("link is a bucket");
+            }
+            const os = info.options.link.bucket !== this.name
+                ? await ObjectStoreImpl.create(this.js, info.options.link.bucket)
+                : this;
+            return os.get(ln);
+        }
+        const d = (0, internal_1.deferred)();
+        const r = {
+            info: new ObjectInfoImpl(info),
+            error: d,
+        };
+        if (info.size === 0) {
+            r.data = emptyReadableStream();
+            d.resolve(null);
+            return Promise.resolve(r);
+        }
+        let controller;
+        const oc = (0, internal_2.consumerOpts)();
+        oc.orderedConsumer();
+        const sha = new internal_1.SHA256();
+        const subj = `$O.${this.name}.C.${info.nuid}`;
+        const sub = await this.js.subscribe(subj, oc);
+        (async () => {
+            for await (const jm of sub) {
+                if (jm.data.length > 0) {
+                    sha.update(jm.data);
+                    controller.enqueue(jm.data);
+                }
+                if (jm.info.pending === 0) {
+                    const hash = sha.digest("base64");
+                    // go pads the hash - which should be multiple of 3 - otherwise pads with '='
+                    const pad = hash.length % 3;
+                    const padding = pad > 0 ? "=".repeat(pad) : "";
+                    const digest = `${exports.digestType}${hash}${padding}`;
+                    if (digest !== info.digest) {
+                        controller.error(new Error(`received a corrupt object, digests do not match received: ${info.digest} calculated ${digest}`));
+                    }
+                    else {
+                        controller.close();
+                    }
+                    sub.unsubscribe();
+                }
+            }
+        })()
+            .then(() => {
+            d.resolve();
+        })
+            .catch((err) => {
+            controller.error(err);
+            d.reject(err);
+        });
+        r.data = new ReadableStream({
+            start(c) {
+                controller = c;
+            },
+            cancel() {
+                sub.unsubscribe();
+            },
+        });
+        return r;
+    }
+    linkStore(name, bucket) {
+        if (!(bucket instanceof ObjectStoreImpl)) {
+            return Promise.reject("bucket required");
+        }
+        const osi = bucket;
+        const { name: n, error } = this._checkNotEmpty(name);
+        if (error) {
+            return Promise.reject(error);
+        }
+        const meta = {
+            name: n,
+            options: { link: { bucket: osi.name } },
+        };
+        return this._put(meta, null);
+    }
+    async link(name, info) {
+        const { name: n, error } = this._checkNotEmpty(name);
+        if (error) {
+            return Promise.reject(error);
+        }
+        if (info.deleted) {
+            return Promise.reject(new Error("src object is deleted"));
+        }
+        if (info.isLink()) {
+            return Promise.reject(new Error("src object is a link"));
+        }
+        const dest = await this.rawInfo(name);
+        if (dest !== null && !dest.deleted) {
+            return Promise.reject(new Error("an object already exists with that name"));
+        }
+        const link = { bucket: info.bucket, name: info.name };
+        const mm = {
+            name: n,
+            bucket: info.bucket,
+            options: { link: link },
+        };
+        await this.js.publish(this._metaSubject(name), JSON.stringify(mm));
+        const i = await this.info(name);
+        return Promise.resolve(i);
+    }
+    async delete(name) {
+        const info = await this.rawInfo(name);
+        if (info === null) {
+            return Promise.resolve({ purged: 0, success: false });
+        }
+        info.deleted = true;
+        info.size = 0;
+        info.chunks = 0;
+        info.digest = "";
+        const jc = (0, internal_1.JSONCodec)();
+        const h = (0, internal_1.headers)();
+        h.set(internal_2.JsHeaders.RollupHdr, internal_2.JsHeaders.RollupValueSubject);
+        await this.js.publish(this._metaSubject(info.name), jc.encode(info), {
+            headers: h,
+        });
+        return this.jsm.streams.purge(this.stream, {
+            filter: this._chunkSubject(info.nuid),
+        });
+    }
+    async update(name, meta = {}) {
+        const info = await this.rawInfo(name);
+        if (info === null) {
+            return Promise.reject(new Error("object not found"));
+        }
+        if (info.deleted) {
+            return Promise.reject(new Error("cannot update meta for a deleted object"));
+        }
+        meta.name = meta.name ?? info.name;
+        const { name: n, error } = this._checkNotEmpty(meta.name);
+        if (error) {
+            return Promise.reject(error);
+        }
+        if (name !== meta.name) {
+            const i = await this.info(meta.name);
+            if (i && !i.deleted) {
+                return Promise.reject(new Error("an object already exists with that name"));
+            }
+        }
+        meta.name = n;
+        const ii = Object.assign({}, info, toServerObjectStoreMeta(meta));
+        // if the name changed, delete the old meta
+        const ack = await this.js.publish(this._metaSubject(ii.name), JSON.stringify(ii));
+        if (name !== meta.name) {
+            await this.jsm.streams.purge(this.stream, {
+                filter: this._metaSubject(name),
+            });
+        }
+        return Promise.resolve(ack);
+    }
+    async watch(opts = {}) {
+        opts.includeHistory = opts.includeHistory ?? false;
+        opts.ignoreDeletes = opts.ignoreDeletes ?? false;
+        let initialized = false;
+        const qi = new internal_1.QueuedIteratorImpl();
+        const subj = this._metaSubjectAll();
+        try {
+            await this.jsm.streams.getMessage(this.stream, { last_by_subj: subj });
+        }
+        catch (err) {
+            if (err.code === "404") {
+                qi.push(null);
+                initialized = true;
+            }
+            else {
+                qi.stop(err);
+            }
+        }
+        const jc = (0, internal_1.JSONCodec)();
+        const copts = (0, internal_2.consumerOpts)();
+        copts.orderedConsumer();
+        if (opts.includeHistory) {
+            copts.deliverLastPerSubject();
+        }
+        else {
+            // FIXME: Go's implementation doesn't seem correct - if history is not desired
+            //  the watch should only be giving notifications on new entries
+            initialized = true;
+            copts.deliverNew();
+        }
+        copts.callback((err, jm) => {
+            if (err) {
+                qi.stop(err);
+                return;
+            }
+            if (jm !== null) {
+                const oi = jc.decode(jm.data);
+                if (oi.deleted && opts.ignoreDeletes === true) {
+                    // do nothing
+                }
+                else {
+                    qi.push(oi);
+                }
+                if (jm.info?.pending === 0 && !initialized) {
+                    initialized = true;
+                    qi.push(null);
+                }
+            }
+        });
+        const sub = await this.js.subscribe(subj, copts);
+        qi._data = sub;
+        qi.iterClosed.then(() => {
+            sub.unsubscribe();
+        });
+        sub.closed.then(() => {
+            qi.stop();
+        }).catch((err) => {
+            qi.stop(err);
+        });
+        return qi;
+    }
+    _chunkSubject(id) {
+        return `$O.${this.name}.C.${id}`;
+    }
+    _metaSubject(n) {
+        return `$O.${this.name}.M.${internal_1.Base64UrlPaddedCodec.encode(n)}`;
+    }
+    _metaSubjectAll() {
+        return `$O.${this.name}.M.>`;
+    }
+    async init(opts = {}) {
+        try {
+            this.stream = objectStoreStreamName(this.name);
+        }
+        catch (err) {
+            return Promise.reject(err);
+        }
+        const max_age = opts?.ttl || 0;
+        delete opts.ttl;
+        // pacify the tsc compiler downstream
+        const sc = Object.assign({ max_age }, opts);
+        sc.name = this.stream;
+        sc.allow_direct = true;
+        sc.allow_rollup_hdrs = true;
+        sc.discard = internal_2.DiscardPolicy.New;
+        sc.subjects = [`$O.${this.name}.C.>`, `$O.${this.name}.M.>`];
+        if (opts.placement) {
+            sc.placement = opts.placement;
+        }
+        if (opts.metadata) {
+            sc.metadata = opts.metadata;
+        }
+        if (typeof opts.compression === "boolean") {
+            sc.compression = opts.compression
+                ? internal_2.StoreCompression.S2
+                : internal_2.StoreCompression.None;
+        }
+        try {
+            await this.jsm.streams.info(sc.name);
+        }
+        catch (err) {
+            if (err.message === "stream not found") {
+                await this.jsm.streams.add(sc);
+            }
+        }
+    }
+    static async create(js, name, opts = {}) {
+        const jsm = await js.jetstreamManager();
+        const os = new ObjectStoreImpl(name, jsm, js);
+        await os.init(opts);
+        return Promise.resolve(os);
+    }
+}
+exports.ObjectStoreImpl = ObjectStoreImpl;
+//# sourceMappingURL=objectstore.js.map