@cascateer/database 0.0.2

package/.github/workflows/publish.yml

@@ -0,0 +1,11 @@
+name: Publish to NPM
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  call-workflow:
+    uses: cascateer/lib/.github/workflows/publish.yml@main
+    secrets: inherit

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@cascateer/database",
+  "version": "0.0.2",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cascateer/database.git"
+  },
+  "scripts": {
+    "semver-patch": "npm version patch && git push origin main --tags",
+    "update": "npm cache clean --force && npx npm-check-updates -u && npm i",
+    "test": "vitest"
+  },
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "dependencies": {
+    "@cascateer/lib": "^1.0.50",
+    "lodash": "^4.18.1",
+    "object-hash": "^3.0.0",
+    "ora": "^9.4.1",
+    "rxjs": "^7.8.2",
+    "uuid": "^14.0.1"
+  },
+  "devDependencies": {
+    "@types/lodash": "^4.17.24",
+    "@types/node": "^26.0.1",
+    "@types/object-hash": "^3.0.6",
+    "vitest": "^4.1.9"
+  }
+}

package/src/defaults.ts

@@ -0,0 +1,4 @@
+export class defaults {
+  static TABLE_BASE_URL = "tables";
+  static FILE_BASE_URL = "files";
+}

package/src/file.ts

@@ -0,0 +1,54 @@
+import { createHash } from "crypto";
+import { createReadStream } from "fs";
+import { Ora } from "ora";
+import { extname, relative, resolve } from "path";
+import { defaults } from "./defaults";
+import { FileTable } from "./types";
+
+export class File {
+  static BASE_URL = defaults.FILE_BASE_URL;
+
+  static fromPath = (path: string) => new File(relative(this.BASE_URL, path));
+
+  static fromUrl = (table: FileTable, url: string, spinner?: Ora) =>
+    table
+      .accessSome([url], spinner)
+      .then(([{ name, checksum }]) => new File(name).verified(checksum));
+
+  constructor(public name: string) {}
+
+  get path() {
+    return resolve(File.BASE_URL, this.name);
+  }
+
+  get extname() {
+    return extname(this.path);
+  }
+
+  async hash() {
+    return new Promise<string>((resolve, reject) => {
+      const hash = createHash("md5");
+
+      createReadStream(this.path)
+        .on("error", (error) => reject(error.message))
+        .on("data", (data) => hash.update(data))
+        .on("end", () => resolve(hash.digest("hex")));
+    });
+  }
+
+  async verify(checksum?: string): Promise<boolean> {
+    return (await this.hash()) === checksum;
+  }
+
+  async verified(checksum: string): Promise<File> {
+    if (await this.verify(checksum)) {
+      return this;
+    }
+
+    throw new Error();
+  }
+
+  toString(): string {
+    return this.path;
+  }
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export { defaults } from "./defaults";
+export { File } from "./file";
+export { createTable } from "./table";

package/src/observables/reduceActions.ts

@@ -0,0 +1,43 @@
+import { property } from "@cascateer/lib";
+import { flatMap } from "@cascateer/lib/observables";
+import { LazyPromise } from "@cascateer/lib/promises";
+import { Function1, last, noop, tap } from "lodash";
+import { mergeAll, OperatorFunction, scan, startWith } from "rxjs";
+import { TableAction, TableActionCreator } from "../types";
+
+export const reduceActions =
+  <R, K extends keyof R>(
+    transform: (records: R[], ...actions: TableAction<R, K>[]) => R[],
+    seed: LazyPromise<
+      R[],
+      {
+        actions: TableAction<R, K>[];
+        callback?: Function1<R[], void>;
+      }
+    >,
+  ): OperatorFunction<TableActionCreator<R, K>, TableAction<R, K>> =>
+  (source) =>
+    source.pipe(
+      startWith(seed),
+      scan(
+        (result, actions) =>
+          result.then(({ records, actions: previousActions }) =>
+            actions.run(records).then(({ actions, callback }) => ({
+              records: tap(transform(records, ...actions), callback ?? noop),
+              actions: actions.map((action, actionIndex, actions) => ({
+                ...action,
+                previousId: (actionIndex >= 1
+                  ? actions[actionIndex - 1]
+                  : last(previousActions)
+                )?.id,
+              })),
+            })),
+          ),
+        Promise.resolve({
+          records: new Array<R>(),
+          actions: new Array<TableAction<R, K>>(),
+        }),
+      ),
+      mergeAll(),
+      flatMap(property("actions")),
+    );

package/src/table.ts

@@ -0,0 +1,284 @@
+import { findDupeBy, nonNullable, nthArg } from "@cascateer/lib";
+import { LazyPromise } from "@cascateer/lib/promises";
+import assert from "assert";
+import { existsSync } from "fs";
+import { mkdir, readdir, readFile, writeFile } from "fs/promises";
+import {
+  difference,
+  fromPairs,
+  intersectionBy,
+  memoize,
+  thru,
+  uniq,
+  without,
+} from "lodash";
+import objectHash from "object-hash";
+import { Ora } from "ora";
+import { resolve } from "path";
+import { mergeMap, NextObserver, Subject, Subscription } from "rxjs";
+import { v4 } from "uuid";
+import { defaults } from "./defaults";
+import { reduceActions } from "./observables/reduceActions";
+import {
+  TableAction,
+  TableActionCreator,
+  TableActionCreatorResult,
+  TableRecordCreator,
+} from "./types";
+
+class Table<R, K extends keyof R> {
+  private static readonly BASE_URL = defaults.TABLE_BASE_URL;
+
+  constructor(
+    public id: string,
+    public key: K,
+    public records: TableRecordCreator<R, K>,
+    private observer: NextObserver<TableActionCreator<R, K>>,
+  ) {}
+
+  get path() {
+    return resolve(Table.BASE_URL, this.id);
+  }
+
+  protected readonly readActions = new LazyPromise<
+    R[],
+    TableActionCreatorResult<R, K>
+  >(async () => {
+    if (!existsSync(this.path)) {
+      await mkdir(this.path, { recursive: true });
+    }
+
+    const files = await readdir(this.path);
+    const actions = new Array<TableAction<R, K>>();
+
+    for (const file of files) {
+      actions.push(
+        await readFile(resolve(this.path, file), "utf-8").then<
+          TableAction<R, K>
+        >(JSON.parse),
+      );
+    }
+
+    const actionsMap = fromPairs(
+      actions.map((action) => [action.previousId ?? "", [action]]),
+    );
+
+    return {
+      actions: actions.reduce(
+        (actions, action) => actions.concat(actionsMap[action.id] ?? []),
+        actionsMap[""] ?? [],
+      ),
+    };
+  });
+
+  applyActions = (records: R[], ...actions: TableAction<R, K>[]) =>
+    actions.reduce((records, action) => {
+      switch (action.type) {
+        case "insert":
+          return records.concat(action.payload.records);
+        case "update": {
+          const targetRecord = action.payload.record;
+
+          return records.map((record) =>
+            this.selectId(record) === this.selectId(targetRecord)
+              ? targetRecord
+              : record,
+          );
+        }
+        case "delete":
+          return without(records, this.selectById(records)(action.payload.id));
+      }
+
+      return records;
+    }, records);
+
+  selectId = (record: R): R[K] => record[this.key];
+  selectById =
+    (records: R[]) =>
+    (id: R[K]): R => (
+      assert(findDupeBy(records, this.selectId) == null),
+      nonNullable(records.find((record) => this.selectId(record) === id))
+    );
+
+  public async dispatch(
+    ...args: NonNullable<TableAction<R, K>["args"]>
+  ): Promise<R[]> {
+    return new Promise<R[]>((callback) => {
+      switch (args[0]) {
+        case "one":
+          this.observer.next(
+            new LazyPromise((records) =>
+              thru(
+                args,
+                ([, id, predicate]) => (
+                  predicate(this.selectById(records)(id)),
+                  {
+                    actions: [],
+                    callback,
+                  }
+                ),
+              ),
+            ),
+          );
+
+          break;
+        case "all":
+          this.observer.next(
+            new LazyPromise((records) =>
+              thru(
+                args,
+                ([, predicate]) => (
+                  predicate(records),
+                  {
+                    actions: [],
+                    callback,
+                  }
+                ),
+              ),
+            ),
+          );
+
+          break;
+        case "insert":
+          this.observer.next(
+            new LazyPromise(async (records) => {
+              const [, predicate] = args;
+
+              const newRecords = await predicate(records.map(this.selectId));
+              const conflictingIds = intersectionBy(
+                newRecords,
+                records,
+                this.selectId,
+              );
+
+              if (conflictingIds.length > 0) {
+                throw new Error(`conflicting ids ${conflictingIds.join(", ")}`);
+              }
+
+              return {
+                actions:
+                  newRecords.length > 0
+                    ? [
+                        {
+                          id: v4(),
+                          type: "insert",
+                          payload: {
+                            records: newRecords,
+                          },
+                        },
+                      ]
+                    : [],
+                callback,
+              };
+            }),
+          );
+
+          break;
+        case "update":
+          this.observer.next(
+            new LazyPromise((records) =>
+              thru(args, async ([, id, predicate]) => {
+                const targetRecord = this.selectById(records)(id);
+                const updatedTargetRecord = await predicate(targetRecord);
+
+                return {
+                  actions:
+                    objectHash(targetRecord ?? null) !==
+                    objectHash(updatedTargetRecord ?? null)
+                      ? [
+                          {
+                            id: v4(),
+                            type: "update",
+                            payload: {
+                              record: updatedTargetRecord,
+                            },
+                          },
+                        ]
+                      : [],
+                  callback,
+                };
+              }),
+            ),
+          );
+
+          break;
+        case "delete":
+          this.observer.next(
+            new LazyPromise((records) =>
+              thru(args, ([, id]) => ({
+                actions: records.some((record) => this.selectId(record) === id)
+                  ? [
+                      {
+                        id: v4(),
+                        type: "delete",
+                        payload: {
+                          id,
+                        },
+                      },
+                    ]
+                  : [],
+                callback,
+              })),
+            ),
+          );
+      }
+    });
+  }
+
+  public async accessSome<A extends R[K][]>(
+    [...ids]: [...A],
+    spinner?: Ora,
+  ): Promise<[...{ [K in keyof A]: R }]> {
+    if (ids.length !== uniq(ids).length) {
+      console.warn(`IDs ${ids.join(", ")} are not unique.`);
+    }
+
+    // @ts-expect-error
+    return this.dispatch(
+      "insert",
+      (currentIds: R[K][]) =>
+        // @ts-expect-error
+
+        this.records(difference(uniq(ids), currentIds), spinner),
+      // @ts-expect-error
+    ).then((records) => ids.map(this.selectById(records)));
+  }
+
+  public async accessAll(): Promise<R[]> {
+    return new Promise<R[]>((resolve) => this.dispatch("all", resolve));
+  }
+}
+
+export { type Table };
+
+export const createTable = memoize(
+  <R, K extends keyof R>(
+    id: string,
+    key: K,
+    records: TableRecordCreator<R, K>,
+  ) =>
+    class TableInstance extends Table<R, K> {
+      private static readonly actions = new Subject<TableActionCreator<R, K>>();
+      private static actionsSubscription?: Subscription;
+
+      constructor() {
+        super(id, key, records, TableInstance.actions);
+
+        TableInstance.actionsSubscription ??= TableInstance.actions
+          .pipe(
+            reduceActions(this.applyActions, this.readActions),
+            mergeMap(async (action) => {
+              const path = resolve(this.path, `${action.id}.json`);
+
+              if (!existsSync(path)) {
+                await writeFile(path, JSON.stringify(action));
+              }
+
+              return action;
+            }),
+          )
+          .subscribe();
+      }
+    },
+  nthArg(0),
+);

package/src/types.ts

@@ -0,0 +1,77 @@
+import { MaybePromise } from "@cascateer/lib";
+import { LazyPromise } from "@cascateer/lib/promises";
+import { Function1 } from "lodash";
+import { Ora } from "ora";
+import { Table } from "./table";
+
+interface BaseTableAction<Type> {
+  id: string;
+  previousId?: string;
+  type: Type;
+  payload: unknown;
+  args?: [Type, ...unknown[]];
+}
+
+interface TableActions<R, K extends keyof R> {
+  one: {
+    payload: never;
+    dispatch: [id: R[K], predicate: Function1<R, void>];
+  };
+  all: {
+    payload: never;
+    dispatch: [predicate: Function1<R[], void>];
+  };
+  insert: {
+    payload: {
+      records: R[];
+    };
+    dispatch: [predicate: Function1<R[K][], MaybePromise<R[]>>];
+  };
+  update: {
+    payload: {
+      record: R;
+    };
+    dispatch: [id: R[K], predicate: Function1<R, MaybePromise<R>>];
+  };
+  delete: {
+    payload: {
+      id: R[K];
+    };
+    dispatch: [id: R[K]];
+  };
+}
+
+export type TableAction<
+  R,
+  K extends keyof R = keyof R,
+  Type extends keyof TableActions<R, K> = keyof TableActions<R, K>,
+> = BaseTableAction<Type> &
+  {
+    [Type in keyof TableActions<R, K>]: {
+      type: Type;
+      payload: TableActions<R, K>[Type]["payload"];
+      args?: [Type, ...TableActions<R, K>[Type]["dispatch"]];
+    };
+  }[Type];
+
+export interface TableActionCreator<R, K extends keyof R> extends LazyPromise<
+  R[],
+  TableActionCreatorResult<R, K>
+> {}
+
+export interface TableActionCreatorResult<R, K extends keyof R> {
+  actions: TableAction<R, K>[];
+  callback?: Function1<R[], void>;
+}
+
+export interface TableRecordCreator<R, K extends keyof R> {
+  (ids: R[K][], spinner?: Ora): MaybePromise<R[]>;
+}
+
+export interface FileTableRecord {
+  originalUrl: string;
+  name: string;
+  checksum: string;
+}
+
+export interface FileTable extends Table<FileTableRecord, "originalUrl"> {}

package/tables/test/2fabb707-99c5-483a-bfaa-e24763c1db25.json

@@ -0,0 +1 @@
+{"id":"2fabb707-99c5-483a-bfaa-e24763c1db25","type":"update","payload":{"record":{"id":"foo","name":"xxx"}},"previousId":"d09834ff-25cd-44fa-a4c2-522547c35866"}

package/tables/test/d09834ff-25cd-44fa-a4c2-522547c35866.json

@@ -0,0 +1 @@
+{"id":"d09834ff-25cd-44fa-a4c2-522547c35866","type":"insert","payload":{"records":[{"id":"foo"}]}}

package/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "module": "commonjs",
+    "types": ["node"],
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "noUncheckedIndexedAccess": true
+  }
+}