keyv-dir-store 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 snomiao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,24 @@
+
+# KeyvDIR
+
+Store key into a file per value
+
+## Store each value with CSV
+
+```ts
+new Keyv({
+  store: new KeyvDirStore("cache/kv", { ext: "csv" }),
+  serialize: ({ value }) => csvFormat(value),
+  deserialize: (str) => ({ value: csvParse(str), expires: undefined }),
+});
+```
+
+## Store each value with YAML
+
+```ts
+new Keyv({
+  store: new KeyvDirStore("cache/kv", { ext: "yaml" }),
+  serialize: ({ value }) => yaml.stringify(value),
+  deserialize: (str) => ({ value: yaml.parse(str), expires: undefined }),
+});
+```

package/index.test.ts

@@ -0,0 +1,55 @@
+import Keyv from "keyv";
+import { KeyvDirStore } from ".";
+
+it("KeyvDirStore works", async () => {
+  // store test
+  const kv = new Keyv<number | string | { obj: boolean }>({
+    store: new KeyvDirStore("cache/test1"),
+    deserialize: KeyvDirStore.deserialize,
+    serialize: KeyvDirStore.serialize,
+  });
+  await kv.clear();
+  await kv.set("a", 1234, -86400e3); // already expired
+  expect(await kv.get("a")).toEqual(undefined); // will delete file before get
+  await kv.set("b", 1234); // never expired
+  expect(await kv.get("b")).toEqual(1234);
+  await kv.set("c", "b", 86400e3); // 1 day
+  expect(await kv.get("c")).toEqual("b");
+  await kv.set("d", { obj: false }, 86400e3); // obj store
+  expect(await kv.get("d")).toEqual({ obj: false });
+
+  // new instance with no cache Obj, to test file cache
+  const kv2 = new Keyv<number | string | { obj: boolean }>({
+    store: new KeyvDirStore("cache/test1"),
+    deserialize: KeyvDirStore.deserialize,
+    serialize: KeyvDirStore.serialize,
+  });
+  expect(await kv2.get("a")).toEqual(undefined); // will delete file before get
+  expect(await kv2.get("b")).toEqual(1234);
+  expect(await kv2.get("c")).toEqual("b");
+  expect(await kv2.get("d")).toEqual({ obj: false });
+});
+it("KeyvDirStore works with only store", async () => {
+  // store test
+  const kv = new Keyv<number | string | { obj: boolean }>({
+    store: new KeyvDirStore("cache/test2"),
+  });
+  await kv.clear();
+  await kv.set("a", 1234, -86400e3); // already expired
+  expect(await kv.get("a")).toEqual(undefined); // will delete file before get
+  await kv.set("b", 1234); // never expired
+  expect(await kv.get("b")).toEqual(1234);
+  await kv.set("c", "b", 86400e3); // 1 day
+  expect(await kv.get("c")).toEqual("b");
+  await kv.set("d", { obj: false }, 86400e3); // obj store
+  expect(await kv.get("d")).toEqual({ obj: false });
+
+  // new instance with no cache Obj, to test file cache
+  const kv2 = new Keyv<number | string | { obj: boolean }>({
+    store: new KeyvDirStore("cache/test2"),
+  });
+  expect(await kv2.get("a")).toEqual(undefined); // will delete file before get
+  expect(await kv2.get("b")).toEqual(1234);
+  expect(await kv2.get("c")).toEqual("b");
+  expect(await kv2.get("d")).toEqual({ obj: false });
+});

package/index.ts

@@ -0,0 +1,116 @@
+import { stat } from "fs/promises";
+import { writeFile } from "fs/promises";
+import { rm } from "fs/promises";
+import { readFile, mkdir } from "fs/promises";
+import { type default as Keyv, type DeserializedData } from "keyv";
+import md5 from "md5";
+import sanitizeFilename from "sanitize-filename";
+import path from "path";
+import { utimes } from "fs/promises";
+type Value = any;
+type CacheMap<Value> = Map<string, DeserializedData<Value>>;
+/**
+ * KeyvDirStore is a Keyv.Store<string> implementation that stores data in files.
+ * @example
+ * const kv = new Keyv<number | string | { obj: boolean }>({
+ *   store: new KeyvDirStore("cache/test"),
+ *   deserialize: KeyvDirStore.deserialize,
+ *   serialize: KeyvDirStore.serialize,
+ * });
+ * await kv.set("a", 1234, -86400e3); // already expired
+ * expect(await kv.get("a")).toEqual(undefined); // will delete file before get
+ * await kv.set("b", 1234); // never expired
+ * expect(await kv.get("b")).toEqual(1234);
+ */
+export class KeyvDirStore implements Keyv.Store<string> {
+  #dir: string;
+  #cache: CacheMap<Value>;
+  #ready: Promise<unknown>;
+  #path: (key: string) => string;
+  ext = ".json";
+  constructor(
+    dir: string,
+    {
+      cache = new Map(),
+      path,
+      ext,
+    }: {
+      cache?: CacheMap<Value>;
+      path?: (key: string) => string;
+      ext?: string;
+    } = {}
+  ) {
+    this.#ready = mkdir(dir, { recursive: true });
+    this.#cache = cache;
+    this.#dir = dir;
+    this.#path = path ?? this.#defaultPath;
+    this.ext = ext ?? this.ext;
+  }
+  #defaultPath(key: string) {
+    // use dir as hash salt to avoid collisions
+    const readableName = sanitizeFilename(key).slice(4, 16);
+    const hashName = md5(this.#dir + key).slice(0, 16);
+    const name = readableName + "-" + hashName + this.ext;
+    return path.join(this.#dir, name);
+  }
+  async get(key: string) {
+    // read memory
+    const cached = this.#cache.get(key);
+    if (cached) {
+      // console.log("memory cache hit but expired", key, cached.expires, Date.now());
+      if (cached.expires && cached.expires < Date.now()) {
+        await this.delete(key);
+      } else {
+        return cached.value;
+      }
+    }
+    // read file cache
+    const stats = await stat(this.#path(key)).catch(() => null);
+    if (!stats) return undefined; // stat not found
+    const expires = +stats.mtime;
+    if (expires !== 0) {
+      const expired = +stats.mtime < +Date.now();
+      if (expired) {
+        //   console.log("file cache hit expired", key, expires, Date.now(), expired);
+        await this.delete(key);
+        return undefined;
+      }
+    }
+    // return this.#parse(await readFile(this.#path(key), "utf8"));
+    return await readFile(this.#path(key), "utf8").catch(() => undefined);
+  }
+  async set(key: string, value: Value, ttl?: number) {
+    if (!value) return await this.delete(key);
+    // const { value, expires } = JSON.parse(stored) as DeserializedData<Value>;
+    const expires = ttl ? Date.now() + ttl : 0;
+    // save to memory
+    this.#cache.set(key, { value, expires });
+    // save to file
+    await this.#ready;
+    // console.log({ key, value, expires });
+    await writeFile(this.#path(key), value); // create a expired file
+    await utimes(this.#path(key), +new Date(), new Date(expires ?? 0)); // set a future expires time (0 as never expired)
+    return true;
+  }
+  async delete(key: string) {
+    // delete memory
+    this.#cache.delete(key);
+    await rm(this.#path(key), { force: true });
+    return true;
+  }
+  async clear() {
+    await rm(this.#dir, { recursive: true }).catch(() => void 0);
+    await mkdir(this.#dir, { recursive: true });
+  }
+  async has(key: string) {
+    return undefined !== (await this.get(key));
+  }
+
+  // Save expires into mtime, and value into file
+  static serialize({ value }: DeserializedData<Value>): string {
+    return JSON.stringify(value, null, 2);
+  }
+  static deserialize(str: string): DeserializedData<Value> {
+    return { value: JSON.parse(str), expires: undefined };
+  }
+}

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "keyv-dir-store",
+  "version": "0.0.1",
+  "author": "snomiao <snomiao@gmail.com>",
+  "type": "module",
+  "exports": {
+    "import": "./dist/index.js",
+    "types": "./index.ts"
+  },
+  "module": "index.ts",
+  "files": [
+    "*.ts",
+    "dist"
+  ],
+  "scripts": {
+    "build": "bun build index.ts --outdir=dist",
+    "release": "bunx standard-version && git push --follow-tags && npm publish",
+    "test": "bun test"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.1.3",
+    "@types/jest": "^29.5.12",
+    "@types/md5": "^2.3.5",
+    "typescript": "^5.4.5"
+  },
+  "peerDependencies": {
+    "keyv": "^4.5.4"
+  },
+  "dependencies": {
+    "md5": "^2.3.0",
+    "sanitize-filename": "^1.6.3"
+  }
+}