@openstax/ts-utils 1.34.0 → 1.34.1

package/packages/utils/src/services/documentStore/unversioned/file-system.ts

@@ -1,194 +0,0 @@
-import * as fsModule from 'fs';
-import path from 'path';
-import { Config, TDocument } from '..';
-import { HashValue, hashValue } from '../../..';
-import { ConfigProviderForConfig, resolveConfigValue } from '../../../config';
-import { ConflictError, NotFoundError } from '../../../errors';
-import { ifDefined, isDefined } from '../../../guards';
-import { assertNoUndefined } from '../fileSystemAssert';
-
-interface Initializer<C> {
-  dataDir: string;
-  fs?: Pick<typeof import('fs'), 'mkdir' | 'readdir' | 'readFile' | 'writeFile'>;
-  configSpace?: C;
-}
-
-export const fileSystemUnversionedDocumentStore = <C extends string = 'fileSystem'>(initializer: Initializer<C>) => <T extends TDocument<T>>() => (configProvider: {[_key in C]: ConfigProviderForConfig<Config>}) => <K extends keyof T>(_: {}, hashKey: K, options?: {afterWrite?: (item: T) => void | Promise<void>; batchAfterWrite?: (items: T[]) => void | Promise<void>; skipAssert?: boolean}) => {
-  const tableName = resolveConfigValue(configProvider[initializer.configSpace || 'fileSystem' as C].tableName);
-  const tablePath = tableName.then((table) => path.join(initializer.dataDir, table));
-  const { mkdir, readdir, readFile, writeFile } = ifDefined(initializer.fs, fsModule);
-  const skipAssert = options?.skipAssert ?? false;
-
-  const mkTableDir = new Promise<void>((resolve, reject) => tablePath.then(
-    (path) => mkdir(path, { recursive: true }, (err) =>
-      err && err.code !== 'EEXIST' ? reject(err) : resolve()
-    )
-  ));
-
-  const hashFilename = (value: HashValue) => `${hashValue(value)}.json`;
-
-  const filePath = async(filename: string) => path.join(await tablePath, filename);
-
-  const load = async(filename: string) => {
-    const path = await filePath(filename);
-    await mkTableDir;
-
-    return new Promise<T | undefined>((resolve, reject) => {
-      readFile(path, (err, readData) => {
-        if (err) {
-          err.code === 'ENOENT' ? resolve(undefined) : reject(err);
-        } else {
-          try {
-            const data = JSON.parse(readData.toString());
-            typeof data === 'object' ? resolve(data) : reject(new Error('unexpected non-object JSON'));
-          } catch (err) {
-            reject(err);
-          }
-        }
-      });
-    });
-  };
-
-  const loadAllDocumentsTheBadWay = async() => {
-    const path = await tablePath;
-    await mkTableDir;
-
-    return new Promise<T[]>((resolve, reject) => readdir(path, (err, files) => err ?
-      reject(err) :
-      Promise.all(files.map((file) => load(file)))
-             .then((allData) => resolve(allData.filter(isDefined)), (err) => reject(err))
-    ));
-  };
-
-  return {
-    loadAllDocumentsTheBadWay,
-    getItemsByField: async(key: keyof T, value: T[K], pageKey?: string) => {
-      if (!skipAssert) assertNoUndefined(value, [key as string]);
-      const pageSize = 10;
-      const items = await loadAllDocumentsTheBadWay();
-      const filteredItems = items.filter((item) => item[key] === value);
-      const startIndex = pageKey ? parseInt(pageKey, 10) : 0;
-      const paginatedItems = filteredItems.slice(startIndex, startIndex + pageSize);
-      const nextPageToken = startIndex + pageSize < filteredItems.length
-        ? (startIndex + pageSize).toString()
-        : undefined;
-      return { items: paginatedItems, nextPageToken };
-    },
-    batchGetItem: async(ids: T[K][]) => {
-      const items = await Promise.all(ids.map((id) => {
-        if (!skipAssert) assertNoUndefined(id, ['id']);
-        return load(hashFilename(id));
-      }));
-      return items.filter(isDefined);
-    },
-    getItem: (id: T[K]) => {
-      if (!skipAssert) assertNoUndefined(id, ['id']);
-      return load(hashFilename(id));
-    },
-    incrementItemAttribute: async(id: T[K], attribute: keyof T) => {
-      if (!skipAssert) assertNoUndefined(id, ['id']);
-      const filename = hashFilename(id);
-      const path = await filePath(filename);
-      await mkTableDir;
-
-      const data = await load(filename);
-      if (!data) { throw new NotFoundError(`Item with ${hashKey.toString()} "${id}" does not exist`); }
-
-      const newValue = typeof data[attribute] === 'number' ? data[attribute] as number + 1 : 1;
-      const newItem = {...data, [hashKey]: id, [attribute]: newValue};
-
-      return new Promise<number>((resolve, reject) => {
-        writeFile(path, JSON.stringify(newItem, null, 2), (err) => err ? reject(err) : resolve(newValue));
-      });
-    },
-    patchItem: async(item: Partial<T>) => {
-      const id = item[hashKey];
-      if (!id) { throw new Error(`Key attribute "${hashKey.toString()}" is required for patchItem`); }
-
-      // This check is just to make this adapter consistent with the dynamo adapter
-      if (Object.keys(item).filter((key) => key !== hashKey.toString()).length === 0) {
-        throw new Error('No attributes to update');
-      }
-
-      const filename = hashFilename(id);
-      const path = await filePath(filename);
-      await mkTableDir;
-
-      const data = await load(filename);
-      if (!data) { throw new NotFoundError(`Item with ${hashKey.toString()} "${id}" does not exist`); }
-
-      const newItem = {...data, ...item};
-      if (!skipAssert) assertNoUndefined(newItem);
-
-      return new Promise<T>((resolve, reject) => {
-        writeFile(path, JSON.stringify(newItem, null, 2), (err) => err ? reject(err) : resolve(newItem));
-      });
-    },
-    putItem: async(item: T) => {
-      if (!skipAssert) assertNoUndefined(item);
-      const path = await filePath(hashFilename(item[hashKey]));
-      await mkTableDir;
-
-      return new Promise<T>((resolve, reject) => {
-        writeFile(path, JSON.stringify(item, null, 2), (err) => err ? reject(err) : resolve(item));
-      });
-    },
-    createItem: async(item: T) => {
-      if (!skipAssert) assertNoUndefined(item);
-      const hashed = hashFilename(item[hashKey]);
-      const existingItem = await load(hashed);
-      if (existingItem) {
-        throw new ConflictError(`Item with ${String(hashKey)} "${item[hashKey]}" already exists`);
-      }
-
-      const path = await filePath(hashed);
-      await mkTableDir;
-
-      return new Promise<T>((resolve, reject) => {
-        writeFile(path, JSON.stringify(item, null, 2), (err) => err ? reject(err) : resolve(item));
-      });
-    },
-    batchCreateItem: async(items: T[], _concurrency = 1): Promise<{successful: T[]; failed: Array<{item: T; error: Error}>}> => {
-      if (items.length === 0) return { successful: [], failed: [] };
-
-      const successful: T[] = [];
-      const failed: Array<{item: T; error: Error}> = [];
-      
-      // Process items sequentially to ensure consistent conflict detection
-      // Note: concurrency parameter is ignored for filesystem to avoid race conditions
-      for (const item of items) {
-        if (!skipAssert) assertNoUndefined(item);
-        try {
-          const hashed = hashFilename(item[hashKey]);
-          const existingItem = await load(hashed);
-          if (existingItem) {
-            throw new ConflictError(`Item with ${String(hashKey)} "${item[hashKey]}" already exists`);
-          }
-
-          const path = await filePath(hashed);
-          await mkTableDir;
-
-          const createdItem = await new Promise<T>((resolve, reject) => {
-            writeFile(path, JSON.stringify(item, null, 2), (err) => err ? reject(err) : resolve(item));
-          });
-
-          successful.push(createdItem);
-          
-          // Only call individual afterWrite if batchAfterWrite is not defined
-          if (!options?.batchAfterWrite) {
-            await options?.afterWrite?.(createdItem);
-          }
-        } catch (error) {
-          failed.push({ item, error: error as Error });
-        }
-      }
-
-      // Call batchAfterWrite with all successful items if defined
-      if (options?.batchAfterWrite && successful.length > 0) {
-        await options.batchAfterWrite(successful);
-      }
-
-      return { successful, failed };
-    },
-  };
-};

package/packages/utils/src/services/documentStore/versioned/README.md

@@ -1,13 +0,0 @@
-# VersionedDocumentStore
-
-a key value document store where all writes are new versions of documents and a version history can be fetched. the interface accepts a `getAuthor` argument at the driver level so you can provide some automatic audit logging for writes, if you don't provide that you have to provide the author information on every call to putItem.
-
-## drivers
-
-### dynamo
-
-production driver, the entity key is the hash key and the version timestamp is the range key, getItem just fetches the last element of the range in one read, while getVersions paginates through the whole range.
-
-### fileSystem
-
-just matches the interface over dumping everything in a json file on disk

package/packages/utils/src/services/documentStore/versioned/dynamodb.spec.ts

@@ -1,376 +0,0 @@
-import * as awsSdk from '@aws-sdk/client-dynamodb';
-import { dynamoVersionedDocumentStore } from './dynamodb';
-import { VersionedDocumentRequiredFields } from '.';
-
-jest.mock('@aws-sdk/client-dynamodb', () => {
-  const sendSpy = jest.fn(() => Promise.resolve()) as any;
-  return {
-    ...jest.requireActual('@aws-sdk/client-dynamodb'),
-    DynamoDB: class {
-      static sendSpy = sendSpy;
-      send(...a: any[]) { return sendSpy(...a); }
-    }
-  };
-});
-
-type TTestDocument = {
-  coolId: string;
-  maybe?: null;
-  name: string;
-  strings?: string[];
-  isCool?: boolean;
-} & VersionedDocumentRequiredFields;
-
-describe('dynamoVersionedDocumentStore', () => {
-  const sendSpy: jest.SpyInstance = (awsSdk.DynamoDB as any).sendSpy;
-  const config = {dynamodb: {tableName: 'cool-table-name'}};
-  const middleware = {};
-  let mockDynamoClient: awsSdk.DynamoDB;
-  let mockClientSend: jest.SpyInstance;
-
-  beforeEach(() => {
-    mockClientSend = jest.fn();
-    mockDynamoClient = {send: mockClientSend} as unknown as awsSdk.DynamoDB;
-  });
-
-  afterEach(() => {
-    jest.clearAllMocks();
-  });
-
-  describe('loadAllDocumentsTheBadWay', () => {
-    it('loads pages of documents', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-
-      sendSpy.mockReturnValueOnce(Promise.resolve({
-        LastEvaluatedKey: {
-          coolId: {S: 'my-document-id'},
-          timestamp: {N: '123'},
-        },
-        Items: [
-          {
-            coolId: {S: 'my-document-id'},
-            timestamp: {N: '123'},
-            author: {M: {type: {S: 'system'}, reason: {S: 'testing'}}},
-            name: {S: 'my-doc-old'},
-          },
-          {
-            coolId: {S: 'my-document-id'},
-            timestamp: {N: '124'},
-            author: {M: {type: {S: 'system'}, reason: {S: 'testing'}}},
-            name: {S: 'my-doc-new'},
-          },
-          {
-            coolId: {S: 'my-document-id'},
-            timestamp: {N: '122'},
-            author: {M: {type: {S: 'system'}, reason: {S: 'testing'}}},
-            name: {S: 'my-doc-old'},
-          }
-        ]
-      }));
-      sendSpy.mockReturnValueOnce(Promise.resolve({
-        Items: [
-          {
-            coolId: {S: 'my-document-id-two'},
-            timestamp: {N: '123'},
-            author: {M: {type: {S: 'system'}, reason: {S: 'testing'}}},
-            name: {S: 'my-doc-two'},
-          }
-        ]
-      }));
-
-      expect(await store.loadAllDocumentsTheBadWay()).toEqual([
-        {
-          coolId: 'my-document-id',
-          timestamp: 124,
-          author: {type: 'system', reason: 'testing'},
-          name: 'my-doc-new',
-        },
-        {
-          coolId: 'my-document-id-two',
-          timestamp: 123,
-          author: {type: 'system', reason: 'testing'},
-          name: 'my-doc-two',
-        }
-      ]);
-    });
-
-    it('handles null items', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-      sendSpy.mockReturnValueOnce(Promise.resolve({
-      }));
-      expect(await store.loadAllDocumentsTheBadWay()).toEqual([
-      ]);
-    });
-  });
-
-  describe('getItem', () => {
-    it('gets document', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-
-      sendSpy.mockReturnValue(Promise.resolve({Items: [{
-        coolId: {S: 'my-document-id'},
-        timestamp: {N: '123'},
-        maybe: {NULL: true},
-        strings: {L: [{S: 'string1'}, {S: 'string2'}]},
-        author: {M: {type: {S: 'system'}, reason: {S: 'testing'}}},
-        name: {S: 'my-doc'},
-        isCool: {BOOL: true}
-      }]}));
-
-      expect(await store.getItem('my-document-id')).toEqual({
-        coolId: 'my-document-id',
-        timestamp: 123,
-        maybe: null,
-        strings: ['string1', 'string2'],
-        author: {type: 'system', reason: 'testing'},
-        name: 'my-doc',
-        isCool: true
-      });
-    });
-
-    it('can get a specific version', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-
-      sendSpy.mockReturnValue(Promise.resolve({Items: [{
-        coolId: {S: 'my-document-id'},
-        timestamp: {N: '123'},
-        maybe: {NULL: true},
-        strings: {L: [{S: 'string1'}, {S: 'string2'}]},
-        author: {M: {type: {S: 'system'}, reason: {S: 'testing'}}},
-        name: {S: 'my-doc'},
-        isCool: {BOOL: true}
-      }]}));
-
-      expect(await store.getItem('my-document-id', 123)).toEqual({
-        coolId: 'my-document-id',
-        timestamp: 123,
-        maybe: null,
-        strings: ['string1', 'string2'],
-        author: {type: 'system', reason: 'testing'},
-        name: 'my-doc',
-        isCool: true
-      });
-    });
-
-    it('gets undefined for missing document', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-      sendSpy.mockReturnValue(Promise.resolve({}));
-      expect(await store.getItem('my-document-id')).toBeUndefined();
-    });
-
-    it('throws on weirdo data types', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-      sendSpy.mockReturnValue(Promise.resolve({Items: [{
-        isCool: {NOT_A_REAL_TYPE: true}
-      }]}));
-
-      await expect(async() => store.getItem('my-document-id')).rejects.toThrow();
-    });
-  });
-
-  describe('putItem', () => {
-    beforeAll(() => {
-      jest.useFakeTimers();
-      jest.setSystemTime(new Date(2020, 6, 28));
-    });
-
-    afterAll(() => {
-      jest.useRealTimers();
-    });
-
-    it('saves document', async() => {
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc', isCool: false, maybe: null, strings: ['string1', 'string2']};
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId');
-      await store.putItem(testDoc, {type: 'system', reason: 'testing'});
-      expect(sendSpy).toHaveBeenCalledWith(expect.objectContaining({
-        input: {
-          TableName: 'cool-table-name',
-          Item: {
-            coolId: {S: 'my-document-id'},
-            timestamp: {N: '1595894400000'},
-            strings: {L: [{S: 'string1'}, {S: 'string2'}]},
-            maybe: {NULL: true},
-            author: {M: {type: {S: 'system'}, reason: {S: 'testing'}}},
-            name: {S: 'my-doc'},
-            isCool: {BOOL: false}
-          }
-        }
-      }));
-    });
-
-    it('calls afterWrite', async() => {
-      const afterWrite = jest.fn();
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc', isCool: false, maybe: null, strings: ['string1', 'string2']};
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', {afterWrite});
-      await store.putItem(testDoc, {type: 'system', reason: 'testing'});
-      expect(afterWrite).toHaveBeenCalledWith(expect.objectContaining({
-        ...testDoc,
-        author: {type: 'system', reason: 'testing'},
-        timestamp: 1595894400000,
-      }));
-    });
-
-    it('returns new document', async() => {
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc'};
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId');
-      const result = await store.putItem(testDoc, {type: 'system', reason: 'testing'});
-      expect(result).toEqual({...testDoc, timestamp: expect.any(Number), author: {type: 'system', reason: 'testing'}});
-    });
-
-    it('throws on weirdo data types', async() => {
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc', foobar: Buffer.from('asdf')};
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-      await expect(async() => store.putItem(testDoc, {type: 'system', reason: 'testing'})).rejects.toThrow();
-    });
-
-    it('uses author provider', async() => {
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc'};
-      const getAuthor = (name: string) => Promise.resolve({type: 'user' as const, name, uuid: 'a uuid'});
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', {getAuthor});
-      const result = await store.putItem(testDoc, 'a name');
-      expect(result).toEqual({...testDoc, timestamp: expect.any(Number), author: {type: 'user', name: 'a name', uuid: 'a uuid'}});
-    });
-  });
-
-  describe('prepareItem', () => {
-    beforeAll(() => {
-      jest.useFakeTimers();
-      jest.setSystemTime(new Date(2020, 6, 28));
-    });
-
-    afterAll(() => {
-      jest.useRealTimers();
-    });
-
-    it('saves document', async() => {
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc', isCool: false, maybe: null, strings: ['string1', 'string2']};
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-      const {save} = await store.prepareItem(testDoc, {type: 'system', reason: 'testing'});
-      await save();
-      expect(sendSpy).toHaveBeenCalledWith(expect.objectContaining({
-        input: {
-          TableName: 'cool-table-name',
-          Item: {
-            coolId: {S: 'my-document-id'},
-            timestamp: {N: '1595894400000'},
-            strings: {L: [{S: 'string1'}, {S: 'string2'}]},
-            maybe: {NULL: true},
-            author: {M: {type: {S: 'system'}, reason: {S: 'testing'}}},
-            name: {S: 'my-doc'},
-            isCool: {BOOL: false}
-          }
-        }
-      }));
-    });
-
-    it('calls afterWrite', async() => {
-      const afterWrite = jest.fn();
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc', isCool: false, maybe: null, strings: ['string1', 'string2']};
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', {afterWrite});
-      const {save} = await store.prepareItem(testDoc, {type: 'system', reason: 'testing'});
-      expect(await save()).toEqual(expect.objectContaining({
-        ...testDoc,
-        author: {type: 'system', reason: 'testing'},
-        timestamp: 1595894400000,
-      }));
-    });
-
-    it('returns new document', async() => {
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc'};
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId');
-      const {document} = await store.prepareItem(testDoc, {type: 'system', reason: 'testing'});
-      expect(document).toEqual({...testDoc, timestamp: expect.any(Number), author: {type: 'system', reason: 'testing'}});
-    });
-
-    it('uses author provider', async() => {
-      const testDoc = {coolId: 'my-document-id', name: 'my-doc'};
-      const getAuthor = (name: string) => Promise.resolve({type: 'user' as const, name, uuid: 'a uuid'});
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', {getAuthor});
-      const {document} = await store.prepareItem(testDoc, 'a name');
-      expect(document).toEqual({...testDoc, timestamp: expect.any(Number), author: {type: 'user', name: 'a name', uuid: 'a uuid'}});
-    });
-  });
-
-  describe('custom client', () => {
-    it('uses provided dynamodb client', async() => {
-      const customSendSpy = jest.fn().mockResolvedValue({Items: [{coolId: {S: 'test-id'}}]});
-      mockDynamoClient.send = customSendSpy;
-
-      const store = dynamoVersionedDocumentStore({dynamoClient: mockDynamoClient})<TTestDocument>()(config)(middleware, 'coolId', undefined);
-      await store.getItem('test-id');
-
-      expect(customSendSpy).toHaveBeenCalled();
-      expect(sendSpy).not.toHaveBeenCalled();
-    });
-  });
-
-  describe('getVersions', () => {
-    it('gets versions for document', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-
-      sendSpy.mockReturnValue(Promise.resolve({Items: [
-        {
-          coolId: {S: 'my-document-id'},
-          timestamp: {N: '123'},
-          name: {S: 'my-doc'}
-        },
-        {
-          coolId: {S: 'my-document-id'},
-          timestamp: {N: '234'},
-          name: {S: 'my-doc'}
-        }
-      ]}));
-
-      expect(await store.getVersions('my-document-id')).toEqual({
-        items: [
-          {coolId: 'my-document-id', name: 'my-doc', timestamp: 123},
-          {coolId: 'my-document-id', name: 'my-doc', timestamp: 234}
-        ]
-      });
-    });
-
-    it('gets next page token', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-
-      sendSpy.mockReturnValue(Promise.resolve({
-        LastEvaluatedKey: {
-          coolId: {S: 'my-document-id'},
-          timestamp: {N: '123'},
-        },
-        Items: [
-        {
-          coolId: {S: 'my-document-id'},
-          timestamp: {N: '234'},
-          name: {S: 'my-doc'}
-        }
-      ]}));
-
-      expect(await store.getVersions('my-document-id')).toEqual(expect.objectContaining({
-        nextPageToken: 123,
-      }));
-    });
-
-    it('gets undefined for unknown document id', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-      sendSpy.mockReturnValue(Promise.resolve({}));
-      expect(await store.getVersions('my-document-id')).toBeUndefined();
-      sendSpy.mockReturnValue(Promise.resolve({Items: []}));
-      expect(await store.getVersions('my-document-id')).toBeUndefined();
-    });
-
-    it('gets version with indicated start point', async() => {
-      const store = dynamoVersionedDocumentStore()<TTestDocument>()(config)(middleware, 'coolId', undefined);
-      sendSpy.mockReturnValue(Promise.resolve({}));
-      await store.getVersions('my-document-id', 123);
-
-      expect(sendSpy).toHaveBeenCalledWith(expect.objectContaining({
-        input: expect.objectContaining({
-          ExclusiveStartKey: {
-            coolId: {S: 'my-document-id'},
-            timestamp: {N: '123'}
-          }
-        })
-      }));
-    });
-  });
-});