@global-cache/core 0.2.3

package/src/server/routes/set.ts

@@ -0,0 +1,27 @@
+import { Express, Router } from 'express';
+import { getConfig } from '../config';
+import { TestRunValueInfo } from '../../shared/value-info';
+import { Setter } from '../setter';
+import { getStorage } from '../storage';
+
+export const router = Router();
+
+export type SetValueParams = {
+  key: string;
+  value?: unknown; // The value to set.
+  error?: string; // An error occured during value computing.
+};
+
+export type SetValueResponse = TestRunValueInfo;
+
+router.post('/run/:runId/set', async (req, res) => {
+  const { runId } = req.params;
+  const { key, value, error } = req.body as SetValueParams;
+  const config = getConfig(req.app as Express);
+
+  const { testRunStorage, persistentStorage } = getStorage(config, runId);
+  const setter = new Setter(testRunStorage, persistentStorage);
+  const valueInfo = await setter.set({ key, value, error });
+
+  res.json(valueInfo);
+});

package/src/server/setter.ts

@@ -0,0 +1,35 @@
+import { setComputed, setError } from '../shared/value-info';
+import { IPersistentStorage, ITestRunStorage } from './storage/types';
+
+export class Setter {
+  constructor(
+    private testRunStorage: ITestRunStorage,
+    private persistentStorage: IPersistentStorage,
+  ) {}
+
+  // eslint-disable-next-line visual/complexity, max-statements
+  async set({ key, value, error }: { key: string; value?: unknown; error?: string }) {
+    const valueInfo = await this.testRunStorage.load(key);
+
+    // todo: convert to warnings?
+    if (!valueInfo) {
+      throw new Error(`Cannot set value for key "${key}" that is not loaded.`);
+    }
+    if (valueInfo.state !== 'computing') {
+      throw new Error(`Cannot set value for key "${key}" that is not in "computing" state.`);
+    }
+
+    if (error) {
+      setError(valueInfo, error);
+      await this.testRunStorage.save(valueInfo, { notify: true });
+      // We delete file on error, as error may be related to the invalid value.
+      if (valueInfo.persistent) await this.persistentStorage.delete(key);
+    } else {
+      setComputed(valueInfo, value);
+      await this.testRunStorage.save(valueInfo, { notify: true });
+      if (valueInfo.persistent) await this.persistentStorage.save(valueInfo);
+    }
+
+    return valueInfo;
+  }
+}

package/src/server/storage/index.ts

@@ -0,0 +1,16 @@
+import { GlobalCacheServerConfigResolved } from '../config';
+import { getFsStorage, getMemoryStorage } from './single-instance';
+
+export function getStorage(
+  { multiInstance, basePath }: GlobalCacheServerConfigResolved,
+  runId: string,
+) {
+  if (multiInstance) {
+    throw new Error('Multi-instance mode is not implemented yet.');
+  }
+
+  return {
+    testRunStorage: getMemoryStorage(runId),
+    persistentStorage: getFsStorage(basePath),
+  };
+}

package/src/server/storage/multi-instance/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Multi-instance mode of Global Storage Server.
+ * - can't reliably write to fs
+ * - can't wait for value via memory, as "set" request can land on another instance
+ * - both non-persistent and persistent values are stored in the external storage (redis/firebase)
+ * - compute-listeners are implemented via redis/firebase notification features
+ */

package/src/server/storage/single-instance/fs.ts

@@ -0,0 +1,64 @@
+/**
+ * File system storage implementation.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { IPersistentStorage } from '../types';
+import { TestRunValueInfo } from '../../../shared/value-info';
+
+export type StoredInfo = {
+  computedAt: number;
+  sig: string;
+  value: unknown;
+};
+
+export class FilesystemStorage implements IPersistentStorage {
+  constructor(private basePath: string) {}
+
+  async load(key: string): Promise<TestRunValueInfo | undefined> {
+    const filePath = this.getFilePath(key);
+    const fileExists = fs.existsSync(filePath);
+    if (!fileExists) return;
+
+    const content = await fs.promises.readFile(filePath, 'utf8');
+    // todo: json parse safe (with warning)
+    const storedInfo: StoredInfo = JSON.parse(content);
+
+    return {
+      key,
+      state: 'computed',
+      persistent: true,
+      computedAt: storedInfo.computedAt,
+      value: storedInfo.value,
+      sig: storedInfo.sig,
+    };
+  }
+
+  async save({ key, value, sig }: TestRunValueInfo) {
+    const computedAt = Date.now();
+    const storedInfo: StoredInfo = { computedAt, sig, value };
+    const content = JSON.stringify(storedInfo, null, 2);
+    const filePath = this.getFilePath(key);
+    await fs.promises.mkdir(path.dirname(filePath), { recursive: true });
+    await fs.promises.writeFile(filePath, content, 'utf8');
+  }
+
+  async delete(key: string) {
+    const filePath = this.getFilePath(key);
+    if (fs.existsSync(filePath)) {
+      await fs.promises.rm(filePath);
+    }
+  }
+
+  private getFilePath(key: string) {
+    return path.join(this.basePath, `${filenamify(key)}.json`);
+  }
+}
+
+// eslint-disable-next-line no-control-regex
+const filenameReservedRegex = /[<>:"/\\|?*\u0000-\u001F]/g;
+function filenamify(s: string, replacement = '-') {
+  return s
+    .replace(filenameReservedRegex, replacement) // prettier-ignore
+    .replace(/\s+/g, replacement); // additionally replace spaces
+}

package/src/server/storage/single-instance/index.ts

@@ -0,0 +1,32 @@
+/**
+ * Single-instance mode of Global Storage Server.
+ * - non-persistent values are stored in memory
+ * - presistent values are stored on file system
+ * - compute-listeners are implemented via memory
+ */
+import { MemoryStorage } from './memory';
+import { FilesystemStorage } from './fs';
+
+// In-memory storage of test runs with values
+const memoryStorageMap = new Map<string, MemoryStorage>();
+const fsStorageMap = new Map<string, FilesystemStorage>();
+
+// todo: cleanup old testRuns from the map
+
+export function getMemoryStorage(runId: string) {
+  let storage = memoryStorageMap.get(runId);
+  if (!storage) {
+    storage = new MemoryStorage();
+    memoryStorageMap.set(runId, storage);
+  }
+  return storage;
+}
+
+export function getFsStorage(basePath: string) {
+  let storage = fsStorageMap.get(basePath);
+  if (!storage) {
+    storage = new FilesystemStorage(basePath);
+    fsStorageMap.set(basePath, storage);
+  }
+  return storage;
+}

package/src/server/storage/single-instance/memory.ts

@@ -0,0 +1,43 @@
+import { TestRunValueInfo } from '../../../shared/value-info';
+import { ITestRunStorage } from '../types';
+
+type Listener = (valueInfo: TestRunValueInfo) => void;
+
+export class MemoryStorage implements ITestRunStorage {
+  private values = new Map<string, TestRunValueInfo>();
+  private listeners = new Map<string /* key */, Listener[]>();
+
+  async load(key: string) {
+    return this.values.get(key);
+  }
+
+  async loadByPrefix(prefix: string) {
+    return [...this.values.values()].filter((valueInfo) => valueInfo.key.startsWith(prefix));
+  }
+
+  async save(valueInfo: TestRunValueInfo, { notify = false } = {}) {
+    this.values.set(valueInfo.key, valueInfo);
+    if (notify) this.notify(valueInfo);
+  }
+
+  async clear() {
+    this.values.clear();
+  }
+
+  async wait(key: string) {
+    return new Promise<TestRunValueInfo>((resolve) => {
+      const listeners = this.listeners.get(key) || [];
+      listeners.push(resolve);
+      this.listeners.set(key, listeners);
+    });
+  }
+
+  private notify(valueInfo: TestRunValueInfo) {
+    const listeners = this.listeners.get(valueInfo.key);
+    try {
+      listeners?.forEach((resolve) => resolve(valueInfo));
+    } finally {
+      this.listeners.delete(valueInfo.key);
+    }
+  }
+}

package/src/server/storage/types.ts

@@ -0,0 +1,13 @@
+import { TestRunValueInfo } from '../../shared/value-info';
+
+export type ITestRunStorage = {
+  load(key: string): Promise<TestRunValueInfo | undefined>;
+  save(valueInfo: TestRunValueInfo, options?: { notify?: boolean }): Promise<void>;
+  wait(key: string): Promise<TestRunValueInfo>;
+};
+
+export type IPersistentStorage = {
+  load(key: string): Promise<TestRunValueInfo | undefined>;
+  save(valueInfo: TestRunValueInfo): Promise<void>;
+  delete(key: string): Promise<void>;
+};

package/src/server/utils/express.ts

@@ -0,0 +1,14 @@
+import express from 'express';
+import http from 'node:http';
+
+/**
+ * Start server.
+ * See: https://github.com/nodejs/node/issues/21482#issuecomment-626025579
+ */
+export async function startExpressServer(app: express.Express, port = 0) {
+  return new Promise<http.Server>((resolve, reject) => {
+    const server = app.listen(port);
+    server.once('listening', () => resolve(server));
+    server.once('error', reject);
+  });
+}

package/src/shared/debug.ts

@@ -0,0 +1,7 @@
+import Debug from 'debug';
+
+export const debug = Debug('global-cache');
+export const debugKey = (key: string, message: string) => {
+  const debug = Debug(`global-cache:${key}`);
+  debug(message);
+};

package/src/shared/logger.ts

@@ -0,0 +1 @@
+export const logger = console;

package/src/shared/sig.ts

@@ -0,0 +1,84 @@
+/**
+ * Calculate the calling signature for a given key:
+ * ttl + stack + computeFn body
+ *
+ * Signature is used to invalidate the key in case of changes in the calling code.
+ * Signature is computed on client and sent to the server.
+ * Server compares the signature with the stored one and invalidates the key if they differ.
+ */
+
+import { TTL } from './ttl';
+
+type SingatureParams = {
+  fn: Function; // eslint-disable-line @typescript-eslint/no-unsafe-function-type
+  stack: string;
+  ttl?: TTL;
+};
+
+type SignatureObj = {
+  fn: string;
+  stack: string;
+  ttl?: TTL;
+};
+
+export type SignatureMismatch = {
+  field: 'fn' | 'stack' | 'ttl' | 'sig';
+  value1: unknown;
+  value2: unknown;
+};
+
+export function calcSignature({ ttl, stack, fn }: SingatureParams) {
+  // todo: use hashing?
+  // todo: remove spaces and newlines from stack and computeFn?
+  const fnBody = fn.toString().replace(/[\n\s]+/g, ' ');
+  const obj: SignatureObj = { ttl, stack, fn: fnBody };
+
+  return JSON.stringify(obj);
+}
+
+// eslint-disable-next-line visual/complexity
+export function checkSignature(
+  key: string,
+  storedSigStr: string,
+  currentSigStr: string,
+): SignatureMismatch | undefined {
+  const storedSig = parseSignatureSafe(storedSigStr);
+  const currentSig = parseSignatureSafe(currentSigStr);
+
+  // fallback to string comparison if parsing failed for any of the signatures
+  // todo: remove in the future releases
+  if (!storedSig || !currentSig) {
+    return storedSigStr !== currentSigStr
+      ? createSignatureMismatch('sig', storedSigStr, currentSigStr)
+      : undefined;
+  }
+
+  // checking stack first -> allows to navigate to the code by click in IDE.
+  if (storedSig.stack !== currentSig.stack) {
+    return createSignatureMismatch('stack', storedSig.stack, currentSig.stack);
+  }
+
+  if (storedSig.ttl !== currentSig.ttl) {
+    return createSignatureMismatch('ttl', storedSig.ttl, currentSig.ttl);
+  }
+
+  if (storedSig.fn !== currentSig.fn) {
+    return createSignatureMismatch('fn', storedSig.fn, currentSig.fn);
+  }
+}
+
+function createSignatureMismatch(
+  field: SignatureMismatch['field'],
+  value1: unknown,
+  value2: unknown,
+): SignatureMismatch {
+  return { field, value1, value2 };
+}
+
+function parseSignatureSafe(sigStr: string) {
+  try {
+    return JSON.parse(sigStr) as SignatureObj;
+  } catch {
+    return null;
+  }
+}

package/src/shared/ttl.ts

@@ -0,0 +1,19 @@
+import ms, { StringValue } from 'ms';
+
+// time to live, can be a number of milliseconds, a string like "1h" or "infinite".
+export type TTL = number | StringValue | 'infinite';
+
+/**
+ * TTL can be a stringified number or a string like "1h" or "infinite".
+ */
+export function parseTTL(ttl: string | number | undefined) {
+  if (ttl === '' || ttl === undefined) return undefined;
+  if (ttl === 'infinite') return -1;
+  const ttlNumber = Number(ttl);
+  return Number.isNaN(ttlNumber) ? ms(ttl as StringValue) : ttlNumber;
+}
+
+export function isExpired(computedAt: number | undefined, ttl: number) {
+  if (!computedAt || ttl === -1) return false;
+  return Date.now() > computedAt + ttl;
+}

package/src/shared/utils/object.ts

@@ -0,0 +1,4 @@
+export function removeUndefined<T extends Record<string, unknown>>(obj?: T): T | undefined {
+  if (!obj) return;
+  return Object.fromEntries(Object.entries(obj).filter(([_, value]) => value !== undefined)) as T;
+}

package/src/shared/value-info.ts

@@ -0,0 +1,40 @@
+/**
+ * Value info in the scope of particular test run.
+ */
+
+export type TestRunValueInfo = {
+  key: string;
+  sig: string;
+  state: 'computing' | 'computed' | 'error';
+  computedAt?: number;
+  value?: unknown;
+  persistent?: boolean; // indicates if the value is stored persistently (e.g., in file system)
+  prevValue?: unknown; // previous value (applicable for persistent keys, used for cleanup)
+  errorMessage?: string; // error message in case of 'error' states
+};
+
+export function updateValueInfo(valueInfo: TestRunValueInfo, props: Partial<TestRunValueInfo>) {
+  return Object.assign(valueInfo, props);
+}
+
+export function setComputed(valueInfo: TestRunValueInfo, value: unknown) {
+  updateValueInfo(valueInfo, {
+    state: 'computed',
+    value,
+    computedAt: Date.now(),
+    errorMessage: undefined,
+  });
+}
+
+export function setError(valueInfo: TestRunValueInfo, errorMessage: string) {
+  updateValueInfo(valueInfo, {
+    state: 'error',
+    errorMessage,
+    value: undefined,
+    computedAt: undefined,
+  });
+}
+
+export function getStaleValue(valueInfo?: TestRunValueInfo | null) {
+  return valueInfo?.persistent ? valueInfo.prevValue : valueInfo?.value;
+}