@openstax/ts-utils 1.32.4 → 1.32.6

package/dist/esm/routing/index.d.ts

@@ -25,6 +25,20 @@ export type RouteMatchRecord<R> = R extends AnyRoute<R> ? {
     route: R;
     params: ParamsForRoute<R>;
 } : never;
+type Flatten<T> = T extends any ? {
+    [K in keyof T]: T[K];
+} : never;
+export type ExternalRoute<R> = R extends Route<infer N, infer P, infer Sa, infer Sr, infer Ri, infer Ro> & infer E ? Route<N, P, Sa, Sr extends {
+    payload: any;
+} ? Flatten<Pick<Sr, 'payload'>> : Record<string, never>, Ri, Ro> & Flatten<Omit<E, 'name' | 'path' | 'handler' | 'requestServiceProvider'>> : never;
+/** this utility simplifies the route type to remove stuff that is only
+ * relevant internal to the route, like the service types, keeping only
+ * the payload type which is necessary for the apiGateway
+ *
+ * this helps avoid the "type too complicated" error that typescript throws
+ * when there are a lot of routes with complex services
+ **/
+export declare const routesList: <R>(routes: R[]) => ExternalRoute<R>[];
 /**
  * The conditional type for the payload for a given route, `R`. This isn't a route structure, its
  * a convention based on the request middleware
@@ -140,7 +154,9 @@ export declare const makeCreateRoute: <Sa, Ri, Ex = {}>() => CreateRoute<Sa, Ri,
  */
 export declare const makeRenderRouteUrl: <Ru extends {
     path: string;
-}>() => <R extends Ru>(route: R, params: ParamsForRoute<R>, query?: QueryParams) => string;
+}>() => <R>(route: ExternalRoute<R> extends Ru ? R & {
+    path: string;
+} : R extends Ru ? R : never, params: ParamsForRoute<R>, query?: QueryParams) => string;
 /**
  * A pre-made result from `makeRenderRouteUrl`, this function interpolates parameter and query
  * arguments into a route path.
@@ -153,7 +169,9 @@ export declare const makeRenderRouteUrl: <Ru extends {
  * @param query the query parameters to add to the route path
  * @returns the interpolated route path
  */
-export declare const renderAnyRouteUrl: <R extends any>(route: R, params: ParamsForRoute<R>, query?: QueryParams) => string;
+export declare const renderAnyRouteUrl: <R>(route: ExternalRoute<R> extends any ? R & {
+    path: string;
+} : R extends any ? R : never, params: ParamsForRoute<R>, query?: QueryParams) => string;
 type RequestPathExtractor<Ri> = (request: Ri) => string;
 type RequestLogExtractor<Ri> = (request: Ri) => JsonCompatibleStruct;
 type RequestRouteMatcher<Ri, R> = (request: Ri, route: R) => boolean;

package/dist/esm/routing/index.js

@@ -2,6 +2,14 @@ import * as pathToRegexp from 'path-to-regexp';
 import queryString from 'query-string';
 import { mapFind, memoize } from '../misc/helpers';
 import { createConsoleLogger } from '../services/logger/console';
+/** this utility simplifies the route type to remove stuff that is only
+ * relevant internal to the route, like the service types, keeping only
+ * the payload type which is necessary for the apiGateway
+ *
+ * this helps avoid the "type too complicated" error that typescript throws
+ * when there are a lot of routes with complex services
+ **/
+export const routesList = (routes) => routes;
 /**
  * Makes a createRoute function that can be used to create routes (this is a factory factory). The
  * `makeCreateRoute` function is typically called once in the backend and once in the frontend to

package/dist/esm/services/apiGateway/index.d.ts

@@ -38,7 +38,7 @@ export type ApiClientResponse<Ro> = Ro extends any ? {
 } : never;
 export type ExpandRoute<T> = T extends ((...args: infer A) => infer R) & {
     renderUrl: (...args: infer Ar) => Promise<string>;
-} ? (...args: A) => R & {
+} ? ((...args: A) => R) & {
     renderUrl: (...args: Ar) => Promise<string>;
 } : never;
 export type MapRoutesToClient<Ru> = [Ru] extends [AnyRoute<Ru>] ? {

package/dist/esm/services/documentStore/unversioned/file-system.d.ts

@@ -8,6 +8,7 @@ interface Initializer<C> {
 export declare const fileSystemUnversionedDocumentStore: <C extends string = "fileSystem">(initializer: Initializer<C>) => <T extends TDocument<T>>() => (configProvider: { [_key in C]: ConfigProviderForConfig<Config>; }) => <K extends keyof T>(_: {}, hashKey: K, options?: {
     afterWrite?: (item: T) => void | Promise<void>;
     batchAfterWrite?: (items: T[]) => void | Promise<void>;
+    skipAssert?: boolean;
 }) => {
     loadAllDocumentsTheBadWay: () => Promise<T[]>;
     getItemsByField: (key: keyof T, value: T[K], pageKey?: string) => Promise<{

package/dist/esm/services/documentStore/unversioned/file-system.js

@@ -6,9 +6,11 @@ import { ConflictError, NotFoundError } from '../../../errors';
 import { ifDefined, isDefined } from '../../../guards';
 import { assertNoUndefined } from '../fileSystemAssert';
 export const fileSystemUnversionedDocumentStore = (initializer) => () => (configProvider) => (_, hashKey, options) => {
+    var _a;
     const tableName = resolveConfigValue(configProvider[initializer.configSpace || 'fileSystem'].tableName);
     const tablePath = tableName.then((table) => path.join(initializer.dataDir, table));
     const { mkdir, readdir, readFile, writeFile } = ifDefined(initializer.fs, fsModule);
+    const skipAssert = (_a = options === null || options === void 0 ? void 0 : options.skipAssert) !== null && _a !== void 0 ? _a : false;
     const mkTableDir = new Promise((resolve, reject) => tablePath.then((path) => mkdir(path, { recursive: true }, (err) => err && err.code !== 'EEXIST' ? reject(err) : resolve())));
     const hashFilename = (value) => `${hashValue(value)}.json`;
     const filePath = async (filename) => path.join(await tablePath, filename);
@@ -43,7 +45,8 @@ export const fileSystemUnversionedDocumentStore = (initializer) => () => (config
     return {
         loadAllDocumentsTheBadWay,
         getItemsByField: async (key, value, pageKey) => {
-            assertNoUndefined(value, [key]);
+            if (!skipAssert)
+                assertNoUndefined(value, [key]);
             const pageSize = 10;
             const items = await loadAllDocumentsTheBadWay();
             const filteredItems = items.filter((item) => item[key] === value);
@@ -56,17 +59,20 @@ export const fileSystemUnversionedDocumentStore = (initializer) => () => (config
         },
         batchGetItem: async (ids) => {
             const items = await Promise.all(ids.map((id) => {
-                assertNoUndefined(id, ['id']);
+                if (!skipAssert)
+                    assertNoUndefined(id, ['id']);
                 return load(hashFilename(id));
             }));
             return items.filter(isDefined);
         },
         getItem: (id) => {
-            assertNoUndefined(id, ['id']);
+            if (!skipAssert)
+                assertNoUndefined(id, ['id']);
             return load(hashFilename(id));
         },
         incrementItemAttribute: async (id, attribute) => {
-            assertNoUndefined(id, ['id']);
+            if (!skipAssert)
+                assertNoUndefined(id, ['id']);
             const filename = hashFilename(id);
             const path = await filePath(filename);
             await mkTableDir;
@@ -97,13 +103,15 @@ export const fileSystemUnversionedDocumentStore = (initializer) => () => (config
                 throw new NotFoundError(`Item with ${hashKey.toString()} "${id}" does not exist`);
             }
             const newItem = { ...data, ...item };
-            assertNoUndefined(newItem);
+            if (!skipAssert)
+                assertNoUndefined(newItem);
             return new Promise((resolve, reject) => {
                 writeFile(path, JSON.stringify(newItem, null, 2), (err) => err ? reject(err) : resolve(newItem));
             });
         },
         putItem: async (item) => {
-            assertNoUndefined(item);
+            if (!skipAssert)
+                assertNoUndefined(item);
             const path = await filePath(hashFilename(item[hashKey]));
             await mkTableDir;
             return new Promise((resolve, reject) => {
@@ -111,7 +119,8 @@ export const fileSystemUnversionedDocumentStore = (initializer) => () => (config
             });
         },
         createItem: async (item) => {
-            assertNoUndefined(item);
+            if (!skipAssert)
+                assertNoUndefined(item);
             const hashed = hashFilename(item[hashKey]);
             const existingItem = await load(hashed);
             if (existingItem) {
@@ -132,7 +141,8 @@ export const fileSystemUnversionedDocumentStore = (initializer) => () => (config
             // Process items sequentially to ensure consistent conflict detection
             // Note: concurrency parameter is ignored for filesystem to avoid race conditions
             for (const item of items) {
-                assertNoUndefined(item);
+                if (!skipAssert)
+                    assertNoUndefined(item);
                 try {
                     const hashed = hashFilename(item[hashKey]);
                     const existingItem = await load(hashed);

package/dist/esm/services/documentStore/versioned/file-system.d.ts

@@ -8,6 +8,7 @@ interface Initializer<C> {
 }
 export declare const fileSystemVersionedDocumentStore: <C extends string = "fileSystem">(initializer: Initializer<C>) => <T extends VersionedTDocument<T>>() => (configProvider: { [_key in C]: ConfigProviderForConfig<Config>; }) => <K extends keyof T, A extends undefined | ((...a: any[]) => Promise<VersionedDocumentAuthor>)>(_: {}, hashKey: K, options?: {
     getAuthor?: A;
+    skipAssert?: boolean;
 }) => {
     loadAllDocumentsTheBadWay: () => Promise<T[]>;
     getVersions: (id: T[K], startVersion?: number) => Promise<{

package/dist/esm/services/documentStore/versioned/file-system.js

@@ -2,7 +2,9 @@ import { assertNoUndefined } from '../fileSystemAssert';
 import { fileSystemUnversionedDocumentStore } from '../unversioned/file-system';
 const PAGE_LIMIT = 5;
 export const fileSystemVersionedDocumentStore = (initializer) => () => (configProvider) => (_, hashKey, options) => {
-    const unversionedDocuments = fileSystemUnversionedDocumentStore(initializer)()(configProvider)({}, 'id');
+    var _a;
+    const skipAssert = (_a = options === null || options === void 0 ? void 0 : options.skipAssert) !== null && _a !== void 0 ? _a : false;
+    const unversionedDocuments = fileSystemUnversionedDocumentStore(initializer)()(configProvider)({ skipAssert: skipAssert }, 'id');
     return {
         loadAllDocumentsTheBadWay: () => {
             return unversionedDocuments.loadAllDocumentsTheBadWay().then(documents => documents.map(document => {
@@ -10,7 +12,8 @@ export const fileSystemVersionedDocumentStore = (initializer) => () => (configPr
             }));
         },
         getVersions: async (id, startVersion) => {
-            assertNoUndefined(id, ['id']);
+            if (!skipAssert)
+                assertNoUndefined(id, ['id']);
             const item = await unversionedDocuments.getItem(id);
             const versions = item === null || item === void 0 ? void 0 : item.items.reverse();
             if (!versions) {
@@ -25,7 +28,8 @@ export const fileSystemVersionedDocumentStore = (initializer) => () => (configPr
             };
         },
         getItem: async (id, timestamp) => {
-            assertNoUndefined(id, ['id']);
+            if (!skipAssert)
+                assertNoUndefined(id, ['id']);
             const item = await unversionedDocuments.getItem(id);
             if (timestamp) {
                 return item === null || item === void 0 ? void 0 : item.items.find(version => version.timestamp === timestamp);
@@ -41,7 +45,8 @@ export const fileSystemVersionedDocumentStore = (initializer) => () => (configPr
                 save: async (changes) => {
                     var _a;
                     const document = { ...item, ...changes, timestamp, author };
-                    assertNoUndefined(document);
+                    if (!skipAssert)
+                        assertNoUndefined(document);
                     const container = (_a = await unversionedDocuments.getItem(document[hashKey])) !== null && _a !== void 0 ? _a : { id: document[hashKey], items: [] };
                     const updated = { ...container, items: [...container.items, document] };
                     await unversionedDocuments.putItem(updated);
@@ -53,7 +58,8 @@ export const fileSystemVersionedDocumentStore = (initializer) => () => (configPr
             var _a;
             const author = (options === null || options === void 0 ? void 0 : options.getAuthor) ? await options.getAuthor(...authorArgs) : authorArgs[0];
             const document = { ...item, timestamp: new Date().getTime(), author };
-            assertNoUndefined(document);
+            if (!skipAssert)
+                assertNoUndefined(document);
             const container = (_a = await unversionedDocuments.getItem(document[hashKey])) !== null && _a !== void 0 ? _a : { id: document[hashKey], items: [] };
             const updated = { ...container, items: [...container.items, document] };
             await unversionedDocuments.putItem(updated);

package/dist/esm/services/fileServer/index.d.ts

@@ -13,6 +13,18 @@ export declare const isFolderValue: (thing: any) => thing is FolderValue;
 export interface FileServerAdapter {
     putFileContent: (source: FileValue, content: string) => Promise<FileValue>;
     getSignedViewerUrl: (source: FileValue) => Promise<string>;
+    getPublicViewerUrl: (source: FileValue) => Promise<string>;
     getFileContent: (source: FileValue) => Promise<Buffer>;
+    getSignedFileUploadConfig: () => Promise<{
+        url: string;
+        payload: {
+            [key: string]: string;
+        };
+    }>;
+    copyFileTo: (source: FileValue, destinationPath: string) => Promise<FileValue>;
+    copyFileToDirectory: (source: FileValue, destinationDirectory: string) => Promise<FileValue>;
+    isTemporaryUpload: (source: FileValue) => boolean;
+    getFileChecksum: (source: FileValue) => Promise<string>;
+    filesEqual: (sourceA: FileValue, sourceB: FileValue) => Promise<boolean>;
 }
 export declare const isFileOrFolder: (thing: any) => thing is FileValue | FolderValue;

package/dist/esm/services/fileServer/localFileServer.js

@@ -1,16 +1,18 @@
 /* cspell:ignore originalname */
+import crypto from 'crypto';
 import fs from 'fs';
 import https from 'https';
 import path from 'path';
 import cors from 'cors';
 import express from 'express';
 import multer from 'multer';
+import { v4 as uuid } from 'uuid';
 import { assertString } from '../../assertions';
 import { resolveConfigValue } from '../../config';
 import { ifDefined } from '../../guards';
-import { once } from '../../misc/helpers';
+import { memoize } from '../../misc/helpers';
 /* istanbul ignore next */
-const startServer = once((port, uploadDir) => {
+const startServer = memoize((port, uploadDir) => {
     // TODO - re-evaluate the `preservePath` behavior to match whatever s3 does
     const upload = multer({ dest: uploadDir, preservePath: true });
     const fileServerApp = express();
@@ -55,6 +57,9 @@ export const localFileServer = (initializer) => (configProvider) => {
     const getSignedViewerUrl = async (source) => {
         return `https://${await host}:${await port}/${source.path}`;
     };
+    const getPublicViewerUrl = async (source) => {
+        return `https://${await host}:${await port}/${source.path}`;
+    };
     const getFileContent = async (source) => {
         const filePath = path.join(await fileDir, source.path);
         return fs.promises.readFile(filePath);
@@ -66,9 +71,55 @@ export const localFileServer = (initializer) => (configProvider) => {
         await fs.promises.writeFile(filePath, content);
         return source;
     };
+    const getSignedFileUploadConfig = async () => {
+        const prefix = 'uploads/' + uuid();
+        return {
+            url: `https://${await host}:${await port}/`,
+            payload: {
+                key: prefix + '/${filename}',
+            }
+        };
+    };
+    const copyFileTo = async (source, destinationPath) => {
+        const sourcePath = path.join(await fileDir, source.path);
+        const destPath = path.join(await fileDir, destinationPath);
+        const destDirectory = path.dirname(destPath);
+        await fs.promises.mkdir(destDirectory, { recursive: true });
+        await fs.promises.copyFile(sourcePath, destPath);
+        return {
+            ...source,
+            path: destinationPath
+        };
+    };
+    const copyFileToDirectory = async (source, destination) => {
+        const destinationPath = path.join(destination, source.label);
+        return copyFileTo(source, destinationPath);
+    };
+    const isTemporaryUpload = (source) => {
+        return source.path.indexOf('uploads/') === 0;
+    };
+    const getFileChecksum = async (source) => {
+        const filePath = path.join(await fileDir, source.path);
+        const fileContent = await fs.promises.readFile(filePath);
+        return crypto.createHash('md5').update(fileContent).digest('hex');
+    };
+    const filesEqual = async (sourceA, sourceB) => {
+        const [aSum, bSum] = await Promise.all([
+            getFileChecksum(sourceA),
+            getFileChecksum(sourceB)
+        ]);
+        return aSum === bSum;
+    };
     return {
         getSignedViewerUrl,
+        getPublicViewerUrl,
         getFileContent,
         putFileContent,
+        getSignedFileUploadConfig,
+        copyFileTo,
+        copyFileToDirectory,
+        isTemporaryUpload,
+        getFileChecksum,
+        filesEqual,
     };
 };

package/dist/esm/services/fileServer/s3FileServer.d.ts

@@ -4,6 +4,7 @@ import { FileServerAdapter } from '.';
 export type Config = {
     bucketName: string;
     bucketRegion: string;
+    publicViewerDomain?: string;
 };
 interface Initializer<C> {
     configSpace?: C;

package/dist/esm/services/fileServer/s3FileServer.js

@@ -1,6 +1,9 @@
 /* cspell:ignore presigner */
-import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import path from 'path';
+import { CopyObjectCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { createPresignedPost } from '@aws-sdk/s3-presigned-post';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { v4 as uuid } from 'uuid';
 import { once } from '../..';
 import { assertDefined } from '../../assertions';
 import { resolveConfigValue } from '../../config';
@@ -9,6 +12,9 @@ export const s3FileServer = (initializer) => (configProvider) => {
     const config = configProvider[ifDefined(initializer.configSpace, 'deployed')];
     const bucketName = once(() => resolveConfigValue(config.bucketName));
     const bucketRegion = once(() => resolveConfigValue(config.bucketRegion));
+    const publicViewerDomain = once(() => 'publicViewerDomain' in config && config.publicViewerDomain
+        ? resolveConfigValue(config.publicViewerDomain)
+        : undefined);
     const s3Service = once(async () => {
         var _a, _b;
         const args = { apiVersion: '2012-08-10', region: await bucketRegion() };
@@ -24,6 +30,10 @@ export const s3FileServer = (initializer) => (configProvider) => {
             expiresIn: 3600, // 1 hour
         });
     };
+    const getPublicViewerUrl = async (source) => {
+        const host = assertDefined(await publicViewerDomain(), new Error(`Tried to get public viewer URL for ${source.path} but no publicViewerDomain configured`));
+        return `https://${host}/${source.path}`;
+    };
     const getFileContent = async (source) => {
         const bucket = await bucketName();
         const command = new GetObjectCommand({ Bucket: bucket, Key: source.path });
@@ -41,9 +51,74 @@ export const s3FileServer = (initializer) => (configProvider) => {
         await (await s3Service()).send(command);
         return source;
     };
+    /*
+     * https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/modules/_aws_sdk_s3_presigned_post.html
+     * https://docs.aws.amazon.com/AmazonS3/latest/userguide/HTTPPOSTExamples.html
+     * https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-post-example.html
+     */
+    const getSignedFileUploadConfig = async () => {
+        const prefix = 'uploads/' + uuid();
+        const bucket = (await bucketName());
+        const Conditions = [
+            { acl: 'private' },
+            { bucket },
+            ['starts-with', '$key', prefix]
+        ];
+        const defaultFields = {
+            acl: 'private',
+        };
+        const { url, fields } = await createPresignedPost(await s3Service(), {
+            Bucket: bucket,
+            Key: prefix + '/${filename}',
+            Conditions,
+            Fields: defaultFields,
+            Expires: 3600, // 1 hour
+        });
+        return {
+            url, payload: fields
+        };
+    };
+    const copyFileTo = async (source, destinationPath) => {
+        const bucket = (await bucketName());
+        const destinationPathWithoutLeadingSlash = destinationPath.replace(/^\//, '');
+        const command = new CopyObjectCommand({
+            Bucket: bucket,
+            Key: destinationPathWithoutLeadingSlash,
+            CopySource: path.join(bucket, source.path),
+        });
+        await (await s3Service()).send(command);
+        return {
+            ...source,
+            path: destinationPathWithoutLeadingSlash
+        };
+    };
+    const copyFileToDirectory = async (source, destination) => {
+        const destinationPath = path.join(destination, source.label);
+        return copyFileTo(source, destinationPath);
+    };
+    const isTemporaryUpload = (source) => {
+        return source.path.indexOf('uploads/') === 0;
+    };
+    const getFileChecksum = async (source) => {
+        const bucket = (await bucketName());
+        const command = new HeadObjectCommand({ Bucket: bucket, Key: source.path });
+        const response = await (await s3Service()).send(command);
+        return assertDefined(response.ETag);
+    };
+    const filesEqual = async (sourceA, sourceB) => {
+        const [aSum, bSum] = await Promise.all([getFileChecksum(sourceA), getFileChecksum(sourceB)]);
+        return aSum === bSum;
+    };
     return {
         getFileContent,
         putFileContent,
         getSignedViewerUrl,
+        getPublicViewerUrl,
+        getSignedFileUploadConfig,
+        copyFileTo,
+        copyFileToDirectory,
+        isTemporaryUpload,
+        getFileChecksum,
+        filesEqual,
     };
 };

package/dist/esm/services/searchProvider/openSearch.js

@@ -26,10 +26,6 @@ export const openSearchService = (initializer = {}) => (configProvider) => {
         maxRetries: 4, // default is 3
         requestTimeout: 5000, // default is 30000
         pingTimeout: 2000, // default is 30000
-        sniffOnConnectionFault: true,
-        sniffOnStart: true,
-        resurrectStrategy: 'ping',
-        agent: { keepAlive: false },
         node: await resolveConfigValue(config.node),
     }));
     return (indexConfig) => {
@@ -65,6 +61,9 @@ export const openSearchService = (initializer = {}) => (configProvider) => {
                 body: params.body,
                 id: params.id,
                 refresh: true
+            }, {
+                requestTimeout: 10000,
+                maxRetries: 1,
             });
         };
         const bulkIndex = async (items) => {
@@ -76,6 +75,9 @@ export const openSearchService = (initializer = {}) => (configProvider) => {
                     item.body
                 ]),
                 refresh: true
+            }, {
+                requestTimeout: 10000,
+                maxRetries: 1,
             });
         };
         const search = async (options) => {