@effectionx/jsonl-store 0.3.0 → 0.4.0

package/fs-helpers.ts

@@ -0,0 +1,129 @@
+import * as fsp from "node:fs/promises";
+import * as path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+/**
+ * Check if a file or directory exists
+ */
+export async function exists(pathOrUrl: string | URL): Promise<boolean> {
+  try {
+    const filePath =
+      pathOrUrl instanceof URL ? fileURLToPath(pathOrUrl) : pathOrUrl;
+    await fsp.access(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Empty a directory by removing all its contents
+ */
+export async function emptyDir(pathOrUrl: string | URL): Promise<void> {
+  const dirPath =
+    pathOrUrl instanceof URL ? fileURLToPath(pathOrUrl) : pathOrUrl;
+
+  try {
+    const entries = await fsp.readdir(dirPath);
+    await Promise.all(
+      entries.map((entry) =>
+        fsp.rm(path.join(dirPath, entry), { recursive: true, force: true }),
+      ),
+    );
+  } catch (error) {
+    // If directory doesn't exist, create it
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      await fsp.mkdir(dirPath, { recursive: true });
+    } else {
+      throw error;
+    }
+  }
+}
+
+export interface WalkEntry {
+  path: string;
+  name: string;
+  isFile: boolean;
+  isDirectory: boolean;
+  isSymlink: boolean;
+}
+
+export interface WalkOptions {
+  includeDirs?: boolean;
+  includeFiles?: boolean;
+  match?: RegExp[];
+}
+
+/**
+ * Walk a directory tree and yield entries
+ */
+export async function* walk(
+  root: string,
+  options: WalkOptions = {},
+): AsyncGenerator<WalkEntry> {
+  const { includeDirs = true, includeFiles = true, match } = options;
+
+  async function* walkDir(dir: string): AsyncGenerator<WalkEntry> {
+    const entries = await fsp.readdir(dir, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+
+      const walkEntry: WalkEntry = {
+        path: fullPath,
+        name: entry.name,
+        isFile: entry.isFile(),
+        isDirectory: entry.isDirectory(),
+        isSymlink: entry.isSymbolicLink(),
+      };
+
+      if (entry.isDirectory()) {
+        if (includeDirs) {
+          if (!match || match.some((re) => re.test(fullPath))) {
+            yield walkEntry;
+          }
+        }
+        yield* walkDir(fullPath);
+      } else if (entry.isFile()) {
+        if (includeFiles) {
+          if (!match || match.some((re) => re.test(fullPath))) {
+            yield walkEntry;
+          }
+        }
+      }
+    }
+  }
+
+  yield* walkDir(root);
+}
+
+/**
+ * Convert a file URL to a path
+ */
+export { fileURLToPath as fromFileUrl };
+
+/**
+ * Convert a path to a file URL
+ */
+export { pathToFileURL as toFileUrl };
+
+/**
+ * Convert a glob pattern to a RegExp
+ * Simplified implementation that handles common patterns
+ */
+export function globToRegExp(
+  glob: string,
+  options?: { globstar?: boolean },
+): RegExp {
+  let pattern = glob
+    // Escape special regex characters except * and ?
+    .replace(/[.+^${}()|[\]\\]/g, "\\$&")
+    // Handle **
+    .replace(/\*\*/g, options?.globstar ? ".*" : "[^/]*")
+    // Handle single *
+    .replace(/(?<!\*)\*(?!\*)/g, "[^/]*")
+    // Handle ?
+    .replace(/\?/g, ".");
+
+  return new RegExp(`^${pattern}$`);
+}

package/jsonl.test.ts

@@ -0,0 +1,135 @@
+import * as fsp from "node:fs/promises";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { beforeEach, describe, it } from "@effectionx/bdd";
+import { each, until } from "effection";
+import { expect } from "expect";
+
+import { JSONLStore } from "./jsonl.ts";
+import type { Store } from "./types.ts";
+
+describe("JSONLStore", () => {
+  let store: Store;
+  let tmpDir: string;
+
+  async function readTmpFile(fileName: string) {
+    return await fsp.readFile(`${tmpDir}/${fileName}`, "utf-8");
+  }
+
+  async function writeTmpFile(fileName: string, data: string) {
+    await fsp.mkdir(join(tmpDir, dirname(fileName)), {
+      recursive: true,
+    });
+    await fsp.writeFile(join(tmpDir, fileName), data, "utf-8");
+  }
+
+  async function appendTmpFile(fileName: string, data: string) {
+    const destination = join(tmpDir, fileName);
+    await fsp.appendFile(destination, data, "utf-8");
+  }
+
+  beforeEach(function* () {
+    tmpDir = yield* until(mkdtemp(join(tmpdir(), "jsonl-test-")));
+    store = JSONLStore.from({ location: tmpDir });
+  });
+
+  describe("from", () => {
+    it("ensures trailing slash for string path", function* () {
+      const store = JSONLStore.from({ location: "/foo" });
+      expect(store.location.protocol).toEqual("file:");
+      expect(store.location.pathname.endsWith("/foo/")).toBe(true);
+    });
+    it("ensures trailing slash for URL", function* () {
+      const store = JSONLStore.from({
+        location: new URL(".cache", "file:///usr/"),
+      });
+      expect(store.location.protocol).toEqual("file:");
+      expect(store.location.pathname.endsWith("/usr/.cache/")).toBe(true);
+    });
+  });
+
+  it("writes to a file", function* () {
+    yield* store.write("hello", "world");
+    expect(yield* until(readTmpFile("hello.jsonl"))).toBe('"world"\n');
+  });
+
+  it("appends to a file", function* () {
+    yield* store.write("hello", "1");
+    yield* store.append("hello", "2");
+    expect(yield* until(readTmpFile("hello.jsonl"))).toBe('"1"\n"2"\n');
+  });
+
+  describe("clearing store", () => {
+    beforeEach(function* () {
+      yield* until(writeTmpFile("hello.jsonl", "world\n"));
+    });
+    it("clears store when called clear", function* () {
+      yield* store.clear();
+      const entries = yield* until(fsp.readdir(tmpDir));
+      expect(entries).toHaveLength(0);
+    });
+  });
+
+  describe("reading content of a file", () => {
+    beforeEach(function* () {
+      yield* until(
+        fsp.writeFile(join(tmpDir, "test.jsonl"), "1\n2\n3\n", "utf-8"),
+      );
+    });
+    it("streams multiple items", function* () {
+      const items: number[] = [];
+      for (const item of yield* each(store.read<number>("test"))) {
+        items.push(item);
+        yield* each.next();
+      }
+      expect(items).toEqual([1, 2, 3]);
+    });
+  });
+
+  describe("checking presence of store", () => {
+    beforeEach(function* () {
+      yield* until(writeTmpFile("1.jsonl", "1\n"));
+    });
+    it("returns true when file exists", function* () {
+      let result: boolean | undefined = undefined;
+      result = yield* store.has("1");
+      expect(result).toBe(true);
+    });
+    it("returns false when file does not exists", function* () {
+      let result: boolean | undefined = undefined;
+      result = yield* store.has("2");
+      expect(result).toBe(false);
+    });
+  });
+
+  describe("finds stored files using glob", () => {
+    beforeEach(function* () {
+      yield* until(writeTmpFile("subdir/1.jsonl", "1\n"));
+      yield* until(writeTmpFile("subdir/2.jsonl", "2\n"));
+      yield* until(writeTmpFile("subdir/3.jsonl", "3\n"));
+    });
+    it("streams multiple items", function* () {
+      const items: number[] = [];
+      for (const item of yield* each(store.find<number>("subdir/*"))) {
+        items.push(item);
+        yield* each.next();
+      }
+      expect(items.sort()).toEqual([1, 2, 3]);
+    });
+
+    describe("multiple values in a single file", () => {
+      beforeEach(function* () {
+        yield* until(appendTmpFile("subdir/2.jsonl", "2.1\n"));
+      });
+      it("streams all lines from globbed files", function* () {
+        const items: number[] = [];
+        for (const item of yield* each(store.find<number>("subdir/*"))) {
+          items.push(item);
+          yield* each.next();
+        }
+        expect(items.sort()).toEqual([1, 2, 2.1, 3]);
+      });
+    });
+  });
+});

package/jsonl.ts

@@ -0,0 +1,254 @@
+import { dirname, join } from "node:path";
+import {
+  emptyDir,
+  exists,
+  fromFileUrl,
+  globToRegExp,
+  toFileUrl,
+  walk,
+} from "./fs-helpers.ts";
+import { JsonParseStream, TextLineStream } from "./stream-helpers.ts";
+
+import {
+  stream,
+  type Operation,
+  type Stream,
+  call,
+  createChannel,
+  createQueue,
+  each,
+  resource,
+  spawn,
+} from "effection";
+import type { Store, StoreConstructorOptions } from "./types.ts";
+
+import fs from "node:fs";
+import * as fsp from "node:fs/promises";
+import { Readable } from "node:stream";
+
+function* mkdir(
+  path: fs.PathLike,
+  options: fs.MakeDirectoryOptions & {
+    recursive: true;
+  },
+): Operation<string | undefined> {
+  return yield* call(() => fsp.mkdir(path, options));
+}
+
+export class JSONLStore implements Store {
+  location: URL;
+  constructor(location: URL) {
+    this.location = location;
+  }
+
+  /**
+   * Creates a store with a location that has a trailing slash.
+   * The trailing slash is important to ensure that the store content
+   * is written to the store directory and not the directory above which
+   * can be very annoying. The location has to be absolute.
+   * ```ts
+   * const store = JSONLStore.from({ location: 'file:///Users/foo/.store/' })
+   * ```
+   *
+   * @param options StoreConstructorOptions
+   * @returns
+   */
+  static from(options: StoreConstructorOptions): JSONLStore {
+    const pathname =
+      options.location instanceof URL
+        ? options.location.pathname
+        : options.location;
+
+    if (pathname.charAt(-1) === "/") {
+      return new JSONLStore(toFileUrl(pathname));
+    }
+    return new JSONLStore(toFileUrl(`${pathname}/`));
+  }
+
+  /**
+   * Returns true when key is present
+   * ```ts
+   * import { useStore } from "jsr:@effectionx/jsonl-store";
+   *
+   * const store = yield* useStore();
+   *
+   * if (yield* store.has("test")) {
+   *  console.log("store exists");
+   * }
+   * ```
+   *
+   * @param key string
+   * @returns boolean
+   */
+  *has(key: string): Operation<boolean> {
+    const location = new URL(`./${key}.jsonl`, this.location);
+
+    return yield* call(async () => {
+      try {
+        return await exists(location);
+      } catch {
+        return false;
+      }
+    });
+  }
+
+  /**
+   * Returns content of a file as a stream
+   *
+   * ```ts
+   * import { each } from "npm:effection@^3";
+   * import { useStore } from "jsr:@effectionx/jsonl-store";
+   *
+   * const store = yield* useStore();
+   *
+   * for (const item of yield* each(store.read<number>("test"))) {
+   *   console.log(item)
+   *   yield* each.next();
+   * }
+   * ```
+   *
+   * @param key string
+   * @returns Stream<T>
+   */
+  read<T>(key: string): Stream<T, void> {
+    const location = new URL(`./${key}.jsonl`, this.location);
+
+    return resource(function* (provide) {
+      const channel = createChannel<T, void>();
+
+      const fileStream = fs.createReadStream(fromFileUrl(location));
+      const webStream = Readable.toWeb(fileStream);
+
+      // biome-ignore lint/suspicious/noExplicitAny: Node stream type incompatibility
+      const lines = (webStream as any)
+        .pipeThrough(new TextDecoderStream())
+        .pipeThrough(new TextLineStream())
+        .pipeThrough(new JsonParseStream());
+
+      yield* spawn(function* () {
+        const reader = lines.getReader();
+        try {
+          while (true) {
+            const { done, value } = yield* call(() => reader.read());
+            yield* channel.send(value as T);
+            if (done) break;
+          }
+        } finally {
+          reader.releaseLock();
+          yield* channel.close();
+        }
+      });
+
+      yield* provide(yield* channel);
+    });
+  }
+
+  /**
+   * Write data to a file, creates the file and necessary directory structure as it goes along.
+   *
+   * ```ts
+   * import { useStore } from "jsr:@effectionx/jsonl-store";
+   *
+   * const store = yield* useStore();
+   * yield* store.write("hello", "world");
+   * ```
+   * @param key string
+   * @param data unknown
+   */
+  *write(key: string, data: unknown): Operation<void> {
+    const location = new URL(`./${key}.jsonl`, this.location);
+
+    yield* mkdir(dirname(fromFileUrl(location)), { recursive: true });
+
+    yield* call(() =>
+      fsp.writeFile(fromFileUrl(location), `${JSON.stringify(data)}\n`, {
+        encoding: "utf-8",
+      }),
+    );
+  }
+
+  /**
+   * Add data to an existing file.
+   *
+   * ```ts
+   * import { useStore } from "jsr:@effectionx/jsonl-store";
+   *
+   * const store = yield* useStore();
+   * yield* store.write("hello", "world");
+   * yield* store.append("hello", "from bob");
+   * ```
+   * @param key string
+   * @param data
+   */
+  *append(key: string, data: unknown): Operation<void> {
+    const location = new URL(`./${key}.jsonl`, this.location);
+
+    yield* call(() =>
+      fsp.appendFile(fromFileUrl(location), `${JSON.stringify(data)}\n`, {
+        encoding: "utf-8",
+      }),
+    );
+  }
+
+  /**
+   * Returns a stream of content from all files matching a glob
+   *
+   * ```ts
+   * import { each } from "npm:effection@^3";
+   * import { useStore } from "jsr:@effectionx/jsonl-store";
+   *
+   * const store = yield* useStore();
+   *
+   * for (const item of yield* each(store.find<number>("subdir/*"))) {
+   *   console.log(item);
+   *    yield* each.next();
+   * }
+   * ```
+   *
+   * @param glob string
+   * @returns Stream<T, void>
+   */
+  find<T>(glob: string): Stream<T, void> {
+    const root = fromFileUrl(this.location);
+
+    const reg = globToRegExp(join(root, glob), {
+      globstar: true,
+    });
+
+    const files = walk(root, {
+      includeDirs: false,
+      includeFiles: true,
+      match: [reg],
+    });
+
+    const read = this.read.bind(this);
+
+    return resource(function* (provide) {
+      const queue = createQueue<T, void>();
+
+      yield* spawn(function* () {
+        for (const file of yield* each(stream(files))) {
+          const key = file.path
+            .replace(root, "")
+            .replaceAll("\\", "/")
+            .replace(/\.jsonl$/, "");
+
+          for (const item of yield* each(read<T>(key))) {
+            queue.add(item);
+            yield* each.next();
+          }
+
+          yield* each.next();
+        }
+
+        queue.close();
+      });
+
+      yield* provide(queue);
+    });
+  }
+
+  *clear(): Operation<void> {
+    yield* call(() => emptyDir(this.location));
+  }
+}

package/mod.ts

@@ -0,0 +1,3 @@
+export { StoreContext, useStore } from "./context.ts";
+export { JSONLStore } from "./jsonl.ts";
+export * from "./types.ts";

package/package.json

@@ -1,30 +1,32 @@
 {
   "name": "@effectionx/jsonl-store",
-  "version": "0.3.0",
+  "version": "0.4.0",
+  "type": "module",
+  "main": "./dist/mod.js",
+  "types": "./dist/mod.d.ts",
+  "exports": {
+    ".": {
+      "development": "./mod.ts",
+      "default": "./dist/mod.js"
+    }
+  },
+  "peerDependencies": {
+    "effection": "^3 || ^4"
+  },
+  "license": "MIT",
   "author": "engineering@frontside.com",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/thefrontside/effectionx.git"
   },
-  "license": "MIT",
   "bugs": {
     "url": "https://github.com/thefrontside/effectionx/issues"
   },
-  "main": "./script/mod.js",
-  "module": "./esm/mod.js",
-  "exports": {
-    ".": {
-      "import": "./esm/mod.js",
-      "require": "./script/mod.js"
-    }
-  },
-  "scripts": {},
   "engines": {
-    "node": ">= 16"
+    "node": ">= 22"
   },
   "sideEffects": false,
-  "dependencies": {
-    "effection": "^3 || ^4.0.0-0"
-  },
-  "_generatedBy": "dnt@dev"
-}
+  "devDependencies": {
+    "@effectionx/bdd": "workspace:*"
+  }
+}

package/stream-helpers.ts

@@ -0,0 +1,45 @@
+/**
+ * A TransformStream that splits input text by newlines
+ */
+export class TextLineStream extends TransformStream<string, string> {
+  #buffer = "";
+
+  constructor() {
+    super({
+      transform: (chunk, controller) => {
+        this.#buffer += chunk;
+        const lines = this.#buffer.split("\n");
+        this.#buffer = lines.pop() ?? "";
+        for (const line of lines) {
+          if (line.length > 0) {
+            controller.enqueue(line);
+          }
+        }
+      },
+      flush: (controller) => {
+        if (this.#buffer.length > 0) {
+          controller.enqueue(this.#buffer);
+        }
+      },
+    });
+  }
+}
+
+/**
+ * A TransformStream that parses each chunk as JSON
+ */
+export class JsonParseStream extends TransformStream<string, unknown> {
+  constructor() {
+    super({
+      transform: (chunk, controller) => {
+        if (chunk.trim().length > 0) {
+          try {
+            controller.enqueue(JSON.parse(chunk));
+          } catch {
+            // Skip invalid JSON lines
+          }
+        }
+      },
+    });
+  }
+}

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "."
+  },
+  "include": ["**/*.ts"],
+  "exclude": ["**/*.test.ts", "dist"],
+  "references": [
+    {
+      "path": "../bdd"
+    }
+  ]
+}

package/types.ts

@@ -0,0 +1,15 @@
+import type { Operation, Stream } from "effection";
+
+export interface Store {
+  location: URL;
+  write(key: string, data: unknown): Operation<void>;
+  append(key: string, data: unknown): Operation<void>;
+  read<T>(key: string): Stream<T, void>;
+  has(key: string): Operation<boolean>;
+  find<T>(glob: string): Stream<T, void>;
+  clear(): Operation<void>;
+}
+
+export interface StoreConstructorOptions {
+  location: URL | string;
+}