@desmat/redis-store 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Mathieu Desjarlais
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,5 @@
+# redis-store
+A lightweight wrapper for using Redis as a data store
+
+## To build
+`npm run build`

package/dist/index.d.ts

@@ -0,0 +1,32 @@
+import { Redis } from "@upstash/redis";
+export type RedisStoreRecord = {
+    id: string;
+    createdAt: number;
+    updatedAt?: number;
+    deletedAt?: number;
+};
+export declare class RedisStore<T extends RedisStoreRecord> {
+    redis: Redis;
+    key: string;
+    setKey: string;
+    valueKey: (id: string) => string;
+    recordOptions: any;
+    debug: boolean;
+    constructor({ url, token, key, setKey, recordOptions, debug, }: {
+        url: string;
+        token: string;
+        key: string;
+        setKey?: string;
+        recordOptions?: any;
+        debug?: boolean;
+    });
+    lookupKeys(value: any, options?: any): any[][];
+    exists(id: string): Promise<boolean>;
+    get(id: string): Promise<T | undefined>;
+    scan(query?: any): Promise<Set<string>>;
+    ids(query?: any): Promise<Set<string>>;
+    find(query?: any): Promise<T[]>;
+    create(value: any, options?: any): Promise<T>;
+    update(value: any, options?: any): Promise<T>;
+    delete(id: string, options?: any): Promise<T>;
+}

package/dist/index.js

@@ -0,0 +1,274 @@
+"use strict";
+/*
+    Some useful commands
+
+    keys *
+    scan 0 match thing:*
+    del thing1 thing2 etc
+    json.get things $
+    json.get things '$[?((@.deletedAt > 0) == false)]'
+    json.get things '$[?((@.deletedAt > 0) == true)]'
+    json.get things '$[?(@.createdBy == "UID")]'
+    json.get things '$[?(@.content ~= "(?i)lorem")]'
+    json.get things '$[?(@.id ~= "(ID1)|(ID2)")]
+    json.set thing:UUID '$.foos[5].bar' '{"car": 42}'
+    json.set thing:UUID '$.foos[1].bar.car' '42'
+    json.get userhaikus '$[?(@.haikuId == "ID" && (@.likedAt > 0) == true)]'
+*/
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisStore = void 0;
+const moment_1 = __importDefault(require("moment"));
+const utils_1 = require("@desmat/utils");
+const redis_1 = require("@upstash/redis");
+class RedisStore {
+    constructor({ url, token, key, setKey, recordOptions, debug, }) {
+        this.redis = new redis_1.Redis({ url, token });
+        this.key = key;
+        this.setKey = setKey || key + "s";
+        this.valueKey = (id) => `${key}:${id}`;
+        this.recordOptions = recordOptions;
+        this.debug = !!debug;
+    }
+    lookupKeys(value, options) {
+        options = { ...this.recordOptions, ...options };
+        this.debug && console.log(`RedisStore.lookupKeys<${this.key}>.lookupKeys`, { value, options });
+        /*
+          create index and lookup sets based on options.lookups
+    
+          given a likedhaiku record:
+          {
+            id: 123:456,
+            userId: 123,
+            haikuId 456,
+          }
+    
+          and lookups:
+          {
+            user: { userId: "haikuId"},
+            haiku: { haikuId: "userId" }
+          }
+      
+          we want indexes:
+    
+          likedhaiku:123:456 -> value (JSON, the rest are sorted sets, not handled here)
+          likedhaikus -> all likedhaiku id's (ie 123:456, etc, not handled here)
+          // NOT SUPPORTED FOR NOW // likedhaikus:users -> all user ids (ie 123, etc) NOTE: this should be a sorted set of user ids with its score as number of haikus liked
+          likedhaikus:user:123 -> all likedhaiku id's for the given user (ie 123:456, etc)
+          // NOT SUPPORTED FOR NOW // likedhaikus:haikus ->  NOTE: this should be a sorted set of haiku ids with its score as number of users who liked it
+          likedhaikus:haiku:456 -> all likedhaiku id's for the given haiku (ie 123:456, etc)
+    
+        */
+        const lookupKeys = !(options === null || options === void 0 ? void 0 : options.noLookup) && Object
+            .entries((options === null || options === void 0 ? void 0 : options.lookups) || {})
+            .map((entry) => {
+            const id = value.id;
+            const lookupName = entry[0];
+            const lookupKey = entry[1];
+            // TODO validate and log errors
+            // @ts-ignore
+            const lookupId = value[lookupKey];
+            // foos:bar:123 -> 123:456
+            return [`${this.setKey}:${lookupName}:${lookupId}`, id];
+        }) || [];
+        this.debug && console.log(`RedisStore<${this.key}>.lookupKeys`, { options, lookupKeys });
+        return lookupKeys;
+    }
+    async exists(id) {
+        this.debug && console.log(`RedisStore<${this.key}>.exists`, { id });
+        const response = await this.redis.exists(this.valueKey(id));
+        this.debug && console.log(`RedisStore<${this.key}>.exists`, { response });
+        return response > 0;
+    }
+    async get(id) {
+        this.debug && console.log(`RedisStore<${this.key}>.get`, { id });
+        const response = await this.redis.json.get(this.valueKey(id), "$");
+        this.debug && console.log(`RedisStore<${this.key}>.get`, { response });
+        let value;
+        if (response && response[0] && !response[0].deletedAt) {
+            value = response[0];
+        }
+        return value;
+    }
+    async scan(query = {}) {
+        this.debug && console.log(`RedisStore<${this.key}>.scan`, { query });
+        const count = query.count;
+        const match = this.valueKey(query.scan);
+        let keys = new Set();
+        let nextCursor = "0";
+        do {
+            const ret = await this.redis.scan(nextCursor, { match, type: "json", count: count - keys.size });
+            this.debug && console.log(`RedisStore<${this.key}>.scan`, { ret });
+            nextCursor = ret[0];
+            ret[1].forEach((key) => keys.size < count && keys.add(key.substring(key.indexOf(':') + 1)));
+        } while (keys.size < count && nextCursor && nextCursor != "0");
+        this.debug && console.log(`RedisStore<${this.key}>.scan`, { keys });
+        return keys;
+    }
+    async ids(query = {}) {
+        this.debug && console.log(`RedisStore<${this.key}>.ids`, { query });
+        if (query.scan) {
+            return this.scan(query);
+        }
+        const min = query.offset || 0;
+        const max = min + (query.count || 0) - 1;
+        delete query.offset;
+        delete query.count;
+        const queryEntries = query && Object.entries(query);
+        // TODO: support more than one
+        if ((queryEntries === null || queryEntries === void 0 ? void 0 : queryEntries.length) > 1) {
+            throw `redis.find(query) only supports a single query entry pair`;
+        }
+        let ids = [];
+        const queryEntry = queryEntries && queryEntries[0];
+        const [queryKey, queryVal] = queryEntry || [];
+        if (queryKey == "id" && Array.isArray(queryVal)) {
+            this.debug && console.log(`RedisStore<${this.key}>.ids special case: query is for IDs`, { ids: queryVal });
+            ids = queryVal;
+        }
+        else {
+            if (queryKey) {
+                /* NOT SUPPORTED FOR NOW
+                if (queryVal == "*") {
+                  // lookup keys via the foos:bars lookup set
+                  keys = (await this.kv.zrange(`${this.setKey}:${queryKey}s`, 0, -1))
+                    // @ts-ignore
+                    .map((key: string) => `${this.key}:${key}`);
+                } else */ if (queryVal) {
+                    // lookup keys via the foos:bar:123 lookup set
+                    // @ts-ignore
+                    ids = await this.redis.zrange(`${this.setKey}:${queryKey}:${queryVal}`, min, max, { rev: true });
+                }
+                else {
+                    throw `redis.find(query) query must have key and value`;
+                }
+                this.debug && console.log(`RedisStore<${this.key}>.ids queried lookup key`, { query, ids });
+            }
+            else {
+                // get all keys via the index set
+                // @ts-ignore
+                ids = await this.redis.zrange(`${this.setKey}`, min, max, { rev: true });
+            }
+        }
+        return new Set(ids);
+    }
+    async find(query = {}) {
+        this.debug && console.log(`RedisStore<${this.key}>.find`, { query });
+        const keys = Array.isArray(query.id)
+            ? Array.from(await this.ids(query))
+                .map((id) => id && this.valueKey(id))
+                .filter(Boolean)
+            : Array.from(await this.ids(query))
+                // @ts-ignore
+                .map((key) => `${this.key}:${key}`);
+        if (keys.length > 100) {
+            console.warn(`RedisStore.RedisStore<${this.key}>.find WARNING: json.mget more than 100 values`, { keys });
+        }
+        else {
+            this.debug && console.log(`RedisStore<${this.key}>.find`, { keys });
+        }
+        // don't mget too many at once otherwise 💥
+        const blockSize = 512;
+        const blocks = keys && keys.length && Array
+            .apply(null, Array(Math.ceil(keys.length / blockSize)))
+            .map((v, block) => (keys || [])
+            .slice(blockSize * block, blockSize * (block + 1)));
+        this.debug && console.log(`RedisStore<${this.key}>.find`, { blocks });
+        const values = blocks && blocks.length > 0
+            ? (await Promise.all(blocks
+                .map(async (keys) => (await this.redis.json.mget(keys, "$"))
+                .flat())))
+                .flat()
+                .filter((value) => value && !value.deletedAt)
+            : [];
+        this.debug && console.log(`RedisStore<${this.key}>.find`, { values });
+        return values;
+    }
+    async create(value, options) {
+        this.debug && console.log(`RedisStore<${this.key}>.create`, { value, options, recordOptions: this.recordOptions });
+        const now = (0, moment_1.default)().valueOf();
+        options = { ...this.recordOptions, ...options };
+        const createdValue = {
+            id: value.id || (0, utils_1.uuid)(),
+            createdAt: value.createdAt || now,
+            ...value,
+        };
+        this.debug && console.log(`RedisStore<${this.key}>.create`, { createdValue });
+        const lookupKeys = this.lookupKeys(createdValue, options);
+        this.debug && console.log(`RedisStore<${this.key}>.create`, { lookupKeys });
+        const responses = await Promise.all([
+            this.redis.json.set(this.valueKey(createdValue.id), "$", createdValue),
+            options.expire && this.redis.expire(this.valueKey(createdValue.id), options.expire),
+            !options.noIndex && this.redis.zadd(this.setKey, { score: createdValue.createdAt, member: createdValue.id }),
+            ...(lookupKeys ? lookupKeys.map((lookupKey) => this.redis.zadd(lookupKey[0], { score: createdValue.createdAt, member: lookupKey[1] })) : []),
+        ]);
+        this.debug && console.log(`RedisStore<${this.key}>.create`, { responses });
+        return createdValue;
+    }
+    async update(value, options) {
+        this.debug && console.log(`RedisStore<${this.key}>.update`, { value, options });
+        if (!value.id) {
+            throw `Cannot update ${this.key}: null id`;
+        }
+        const prevValue = await this.get(value.id);
+        if (!prevValue) {
+            throw `Cannot update ${this.key}: does not exist: ${value.id}`;
+        }
+        const now = (0, moment_1.default)().valueOf();
+        options = { ...this.recordOptions, ...options };
+        const updatedValue = {
+            ...value,
+            updatedAt: now,
+        };
+        // optionally update lookups 
+        // @ts-ignore
+        const prevLookupKeys = new Map(this.lookupKeys(prevValue, options));
+        // @ts-ignore
+        const lookupKeys = new Map(this.lookupKeys(updatedValue, options));
+        const lookupsToRemove = prevLookupKeys && Array.from(prevLookupKeys)
+            .filter(([k, v]) => !lookupKeys || lookupKeys.get(k) != v);
+        const lookupsToAdd = lookupKeys && Array.from(lookupKeys)
+            .filter(([k, v]) => !prevLookupKeys || prevLookupKeys.get(k) != v);
+        this.debug && console.log(`RedisStore<${this.key}>.update`, { prevLookupKeys, lookupKeys, prevLookupKeyMap: prevLookupKeys, keysToRemove: lookupsToRemove, keysToAdd: lookupsToAdd });
+        if (lookupsToRemove && lookupsToRemove.length) {
+            this.debug && console.log(`RedisStore<${this.key}>.update deleting previous lookup keys`, { prevLookupKeys });
+            const response = await Promise.all([
+                ...lookupsToRemove.map((lookupKey) => this.redis.zrem(lookupKey[0], lookupKey[1]))
+            ]);
+            this.debug && console.log(`RedisStore<${this.key}>.update deleted previous lookup keys`, { response });
+        }
+        const response = await Promise.all([
+            this.redis.json.set(this.valueKey(value.id), "$", updatedValue),
+            options.expire && this.redis.expire(this.valueKey(value.id), options.expire),
+            ...(lookupsToAdd ? lookupsToAdd.map((lookupKey) => this.redis.zadd(lookupKey[0], { score: updatedValue.createdAt || updatedValue.updatedAt, member: lookupKey[1] })) : []),
+        ]);
+        this.debug && console.log(`RedisStore<${this.key}>.update`, { response });
+        return updatedValue;
+    }
+    async delete(id, options = {}) {
+        this.debug && console.log(`RedisStore<${this.key}>.delete`, { id, options });
+        if (!id) {
+            throw `Cannot delete ${this.key}: null id`;
+        }
+        options = { ...this.recordOptions, ...options };
+        const value = await this.get(id);
+        if (!value) {
+            throw `Cannot update ${this.key}: does not exist: ${id}`;
+        }
+        const lookupKeys = this.lookupKeys(value, options);
+        this.debug && console.log(`RedisStore<${this.key}>.delete`, { lookupKeys });
+        value.deletedAt = (0, moment_1.default)().valueOf();
+        const response = await Promise.all([
+            options.hardDelete
+                ? this.redis.json.del(this.valueKey(id), "$")
+                : this.redis.json.set(this.valueKey(id), "$", { ...value, deletedAt: (0, moment_1.default)().valueOf() }),
+            this.redis.zrem(this.setKey, id),
+            ...(lookupKeys ? lookupKeys.map((lookupKey) => this.redis.zrem(lookupKey[0], lookupKey[1])) : []),
+        ]);
+        this.debug && console.log(`RedisStore<${this.key}>.delete`, { response });
+        return value;
+    }
+}
+exports.RedisStore = RedisStore;

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@desmat/redis-store",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/desmat/redis-store.git"
+  },
+  "keywords": [
+    "utility",
+    "library",
+    "redis",
+    "store"
+  ],
+  "author": "Mathieu Desjarlais",
+  "license": "MIT",
+  "dependencies": {
+    "@desmat/utils": "^1.0.0",
+    "@upstash/redis": "^1.34.3"
+  },
+  "devDependencies": {
+    "typescript": "^4.0.0"
+  }
+}