@openstax/ts-utils 1.33.1 → 1.34.0

package/packages/lambda/webpack.config.js

@@ -0,0 +1,97 @@
+/* spell-checker: ignore Builtins */
+const fs = require('fs');
+const path = require('path');
+const {assertDefined} = require('@openstax/ts-utils/assertions');
+const {ENV_BUILD_CONFIGS} = require('@openstax/ts-utils/config');
+const nodeBuiltins = require('builtin-modules');
+const TerserPlugin = require('terser-webpack-plugin');
+const webpack = require('webpack');
+const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
+
+
+// we need to run the code to get the ENV configs, so build it now
+// to make sure its up to date
+require('child_process').execSync(
+  './script/build.bash --clean',
+  {stdio: 'inherit'}
+);
+
+const lambdaDir = 'build/src/functions';
+const lambdaNames = fs.readdirSync(path.join(__dirname, lambdaDir));
+
+const externals = ['aws-sdk', 'aws-crt']
+  .concat(nodeBuiltins)
+  .reduce((externalsMap, moduleName) => {
+    externalsMap[moduleName] = moduleName;
+    return externalsMap;
+  }, {});
+
+module.exports = lambdaNames.map(lambdaName => {
+  const lambdaPath = path.join(__dirname, lambdaDir, lambdaName);
+  const entryPath = path.join(lambdaPath, 'entry', 'lambda');
+  const webpackConfigPath = path.join(entryPath, 'webpack.config.js');
+  const outputPath = path.join(__dirname, 'dist', lambdaName);
+  const services = path.join(entryPath, 'services');
+  const routes = path.join(lambdaPath, 'core', 'routes');
+
+  // trying to cover as much as possible without actually wiring up the app
+  // to catch all `envConfig` calls without trying to run them.
+  require(services)
+
+  // require routes if they exist
+  if (fs.existsSync(routes)) { require(routes) }
+
+  const env = ENV_BUILD_CONFIGS.splice(0, ENV_BUILD_CONFIGS.length);
+
+  if (fs.existsSync(webpackConfigPath)) { return require(webpackConfigPath)({lambdaName, entryPath, outputPath, env}) }
+
+  return {
+    entry: ['source-map-support/register', entryPath],
+    externals,
+
+    output: {
+      path: outputPath,
+      libraryTarget: 'commonjs',
+      filename: 'index.js',
+    },
+
+    target: 'node',
+    mode: 'production',
+
+    optimization: {
+      concatenateModules: true,
+      minimize: true,
+      minimizer: [new TerserPlugin({ terserOptions: {
+          mangle: false
+        },
+      })],
+    },
+
+    module: {
+      rules: [
+        {
+          test: /\.ts$/,
+          use: 'ts-loader',
+          exclude: [], // include node modules
+        }
+      ]
+    },
+
+    resolve: {
+      extensions: ['.ts', '.js'],
+    },
+
+    devtool: 'source-map',
+
+    plugins: [
+      new webpack.DefinePlugin({
+        'typeof __PROCESS_ENV': JSON.stringify('object'),
+        __PROCESS_ENV: JSON.stringify(Object.fromEntries(env.map(key => {
+          return [key, assertDefined(process.env[key], `missing environment key ${key}`)]
+        })))
+      }),
+      new webpack.ProgressPlugin(),
+      ...(lambdaName === process.env.DEBUG_MODULES ? [new BundleAnalyzerPlugin()] : []),
+    ],
+  }
+});

package/packages/utils/.eslintrc.js

@@ -0,0 +1,64 @@
+module.exports = {
+  root: true,
+  env: { node: true, es6: true },
+  parserOptions: {
+    ecmaVersion: 2018
+  },
+  overrides: [
+    {
+      files: ['**/*.ts?(x)'],
+      parser: '@typescript-eslint/parser',
+      parserOptions: {
+        project: 'tsconfig.json',
+        tsconfigRootDir: __dirname,
+        ecmaVersion: 2018,
+        sourceType: 'module',
+        warnOnUnsupportedTypeScriptVersion: true
+      },
+      plugins: ['@typescript-eslint'],
+      extends: [
+        'eslint:recommended',
+        'plugin:import/typescript'
+      ],
+      rules: {
+        '@typescript-eslint/ban-ts-comment': 'off',
+        '@typescript-eslint/no-inferrable-types': [
+          'warn',
+          { ignoreParameters: true }
+        ],
+        semi: 'off',
+        '@typescript-eslint/semi': ['warn'],
+        '@typescript-eslint/member-delimiter-style': ['warn'],
+        'no-unused-vars': 'off',
+        '@typescript-eslint/no-unused-vars': [
+          'warn',
+          {
+            args: 'none',
+            ignoreRestSiblings: true,
+            varsIgnorePattern: '^_'
+          }
+        ],
+        '@typescript-eslint/no-explicit-any': 'off'
+      }
+    }
+  ],
+  ignorePatterns: ['node_modules', 'dist', 'coverage'],
+  plugins: ['import', 'jest'],
+  extends: [
+    'eslint:recommended',
+    'plugin:jest/recommended',
+    'plugin:jest/style'
+  ],
+  rules: {
+    quotes: ['warn', 'single'],
+    'quote-props': ['warn', 'as-needed'],
+    'import/order': [
+      'warn',
+      { 'newlines-between': 'never', alphabetize: { order: 'asc' } }
+    ],
+    'sort-imports': [
+      'warn',
+      { ignoreDeclarationSort: true, ignoreCase: true }
+    ]
+  }
+}

package/packages/utils/README.md

@@ -0,0 +1,118 @@
+# @openstax/ts-utils
+
+A collection of utilities typescript applications. browser and server support.
+
+## Libraries
+overviews of the libraries provided and their purposes, for more specific information check out the comments in the source code.
+
+### [types.ts](./types.ts)
+utility types, no javascript functionality in here. A dumping ground for commonly used utility types or sorcery copied of stack overflow.
+
+### [guards.ts](./guards.ts)
+generic [type guards](https://www.typescriptlang.org/docs/handbook/advanced-types.html#user-defined-type-guards). most of this isn't super interesting its just annoying to have to re-define these in every project.
+
+### [assertions.ts](./assertions.ts)
+like a guard but throws if the condition fails. you can use an assertion if you're extra sure that something is true but typescript isn't so sure, its possible that a bug might exist but normally your expected case should always be the case. these assertions allow you to have useful errors/messages for the exceptional bug, without generating conditional logic in your application code. this is almost entirely driven by not wanting to create code branches that will then need unit tests.
+
+### [index.ts](./index.ts)
+basic array/object access/manipulation utilities
+
+### [errors.ts](./errors.ts)
+a set of common application errors for the other libraries to use
+
+### [fetch.ts](./fetch.ts)
+utilities for working with `fetch` and fetching data. mostly there are constructors and guards for working with response objects that capture both a request status and associated data. the actual fetch api only throws if there is a connection errors, so most data access utilities need a way to communicate application layer failures back to the client in a discernible way. you can _almost_ do this with regular response objects, but then you're forcing all of the response processing down into user-land, and this system provides an additional loading state for UIs.
+
+### [config.ts](./config.ts)
+defines ConfigProvider as an arbitrarily nested object where leaves must be ConfigValueProviders. ConfigValueProviders are strings, promises of strings, functions without arguments that return strings, or promises of strings. the library has a utility to resolve a ConfigValueProvider to a string, and a few re-usable providers.
+
+### [middleware.ts](./middleware.ts)
+middleware are chains of functions, with shared access to a parent application or services, that modify or decorate a result. middleware chains are used to pre-process requests before they reach routing, post-process results after routing, and compose service providers for the routes to use.
+
+### [routing.ts](./routing.ts)
+a flexible routing library with utils for defining routes, matching requests to routes, and rendering urls for routes. strong typing for route params, request bodies, response bodies. reverse routing asserts not only that the correct route and params are given, but that the route is composed in the application. built to be implementation agnostic and support both browser SPA routing and server side API routing.
+
+### [pagination.ts](./pagination.ts)
+utilities for interpreting pagination requests and generating pagination responses for both 'load more' and 'page number' style pagination. included middleware for working with the routing library.
+
+## Services
+Services differ from the rest of the libraries in that they have a single discrete interface and support multiple implementations of that interface.
+
+### [apiGateway](./services/apiGateway/index.ts)
+dynamically generates an api client from route configuration. if you're using the routing library in the api, this allows you to import your route types directly into your client and have very strong typing on your api calls for free. includes error handling, filtering responses by status code, and narrowing the expected data type by given status code.
+
+### [authProvider](./services/authProvider)
+integrates with openstax.org/accounts, includes several implementations for both browser and server side integrations.
+
+### [lrsGateway](./services/lrsGateway/index.ts)
+handles authentication and payload formatting for interacting with the LRS
+
+### [versionedDocumentStore](./services/versionedDocumentStore)
+a key/value document store with integrated audit logging. all writes are done as new versions of documents and version history can be fetched. includes implementations for dynamodb and local file system.
+
+### [searchProvider](./services/searchProvider/index.ts)
+allows filtering and open text search for documents. has strong typing for filterable values based on the document type definition. interface is based on elasticsearch options, but currently we only have an in memory filtering implementation. works out of the box with versionedDocumentStore.
+
+## Notes
+
+### Service drivers and configuration
+Service providers are defined in a little bit of a weird nested way, this is supposed to help separate driver specific and instance specific configurations, and let you swap drivers in different application entry points without too many application dependencies.
+
+an entry point is the file you point to when you build or run the application. the idea is that you make a different entry point files for different environments or contexts. so you have one for launching local development and one for launching an all-in-one stack on a testing server and another one for launching a production environment. in order to reduce environment specific issues you reduce your entry point to be as small as possible and just assemble the required bits for that environment to work. in the recommended setup, one of the main things the entry point does is choose which service provider drivers to use. there are two main benefits of using entry points, its more declarative than using downstream conditional logic with some kind of `driver: fs` flag in the config, and it allows bundled javascript to only include the dependencies relevant for the drivers that are being used in that build.
+
+the idea is that you may have some entities you're putting in a VersionedDocumentStore and maybe some others in a RelationalEntityStore but you're for sure not putting some versioned entities in dynamo and some in a local json file. so in your development environment entry point you provide your config for `versionedDocumentStore: fileSystemVersionedDocumentStore` and in the deployed environment entry point you put `versionedDocumentStore: dynamoVersionedDocumentStore`.
+
+there are two levels the drivers are configured at, one is the driver level and the other is the instance level. for the VersionedDocumentStore the fileSystem driver takes a path to the directory to store the files in. the instance level may vary based on the usage of that driver, for example if you're storing 4 different entities in VersionedDocumentStores, you need to provide different table names to the driver to configure each store to create separate instances of that store. it just happens that for VersionedDocumentStore the fileSystem and dynamo implementations use the same instance config, this isn't always the case. once you get into your application logic you won't know which entry point was used to bootstrap your application, so you'll just know the driver is _one of_ the possible versions, typescript will recognize this and throw an error unless you provide a config that would work for any of the possible drivers. in order to make this more legible (and separate key collisions), most of the services can be set up in the entry point with a config prefix, so when you get into application-land it looks more like you're providing two discrete configs.
+
+for example:
+
+```typescript
+// entry/anEntryPoint.ts
+export const lambdaServices = {
+  // in this entry we provide the dynamo driver
+  versionedDocumentStore: dynamoVersionedDocumentStore({configSpace: 'dynamodb'}),
+};
+
+// entry/anotherEntryPoint.ts
+export const localServices = {
+  // in this entry we provide the fileSystem driver, it has a different config prefix
+  versionedDocumentStore: fileSystemVersionedDocumentStore({dataDir, configSpace: 'fileSystem'}),
+};
+
+// core/types.ts
+
+// there is other wiring up that happens with the app services on the request responder, if
+// you made a new entrypoint and tried to use a different format of services without registering it
+// here it would yell at you.
+export type AppServices = LambdaServices | LocalServices;
+
+// myCoolDocumentStoreMiddleware.ts
+
+// this config satisfies both config drivers with separate sections for each one.
+// the envConfig is defined here, but it wouldn't try to actually resolve that value
+// unless it was being used.
+const config = {
+  fileSystem: {
+    tableName: 'coolDocument'
+  },
+  dynamodb: {
+    tableName: envConfig('TestDocumentTableName', 'runtime'),
+  }
+};
+
+export const myCoolDocumentStoreMiddleware = <M extends {}>(app: AppServices) => {
+  // providing the document interface and the config prepares the driver to work for this entity in particular.
+  // the extra function call in this line is to work around some typescript quirkiness with generic type inference.
+  // this line is only run once when the app is wired up, doing this call here allows the driver to do internal memory caching
+  // if it wants to.
+  const makeCoolDocumentStore = app.versionedDocumentStore<CoolDocument>()(config);
+
+  return (middleware: M) => ({
+    ...middleware,
+    // this part is run for every api request processed. in this case there it could have been combined with the call above,
+    // but most of the time you'll want to pull in the authProvider and hook up automatic audit logging, and that will be different on 
+    // each request. look at the VersionedDocumentStore documentation for details.
+    coolDocumentStore: makeCoolDocumentStore('id'),
+  })
+};
+```

package/packages/utils/jest-global-setup.js

@@ -0,0 +1,3 @@
+module.exports = async () => {
+  process.env.TZ = 'UTC';
+};

package/packages/utils/jest.config.js

@@ -0,0 +1,25 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  globalSetup: './jest-global-setup.js',
+  testMatch: [
+    '**/src/**/?(*.)+(spec).ts'
+  ],
+  collectCoverageFrom: [
+    'src/**/*.{js,ts}',
+  ],
+  coverageReporters: [
+    'text-summary',
+    'html',
+    'lcovonly'
+  ],
+  coverageThreshold: {
+    global: {
+      branches: 100,
+      functions: 100,
+      lines: 100,
+      statements: 100
+    }
+  },
+  resolver: './jest.resolver.js'
+};

package/packages/utils/jest.resolver.js

@@ -0,0 +1,17 @@
+// https://github.com/cloudflare/miniflare/issues/271#issuecomment-1255544333
+//
+// the uuid package says that it fixed this in v9 here https://github.com/uuidjs/uuid/pull/616#issuecomment-1237428554
+// but i tired it and was still getting the error. try removing this hack in the future and
+// see if it got figured out.
+module.exports = (path, options) => {
+  return options.defaultResolver(path, {
+    ...options,
+    packageFilter: pkg => {
+      if (pkg.name === 'uuid') {
+        delete pkg['exports'];
+        delete pkg['module'];
+      }
+      return pkg;
+    },
+  });
+};

package/packages/utils/package.json

@@ -0,0 +1,238 @@
+{
+  "name": "@openstax/ts-utils",
+  "version": "1.33.0",
+  "bin": {
+    "ts-utils": "./script/bin-entry.bash"
+  },
+  "dependencies": {
+    "@aws-sdk/client-dynamodb": "^3.171.0",
+    "@aws-sdk/client-s3": "^3.267.0",
+    "@aws-sdk/client-ssm": "^3.171.0",
+    "@aws-sdk/client-sts": "^3.171.0",
+    "@aws-sdk/s3-presigned-post": "^3.267.0",
+    "@aws-sdk/s3-request-presigner": "^3.267.0",
+    "aws-sdk": "^2.1145.0",
+    "cookie": "^0.5.0",
+    "date-fns": "^2.29.2",
+    "deep-equal": "^2.1.0",
+    "jsonwebtoken": "^9.0.0",
+    "jwks-rsa": "^3.1.0",
+    "ms": "^2.1.3",
+    "node-jose": "^2.2.0",
+    "path-to-regexp": "^6.2.0",
+    "query-string": "^7.1.1",
+    "tiny-async-pool": "^2.1.0",
+    "uuid": "^9.0.0"
+  },
+  "devDependencies": {
+    "@opensearch-project/opensearch": "^1.2.0",
+    "@types/cookie": "^0.5.0",
+    "@types/cors": "^2.8.13",
+    "@types/deep-equal": "^1.0.1",
+    "@types/express": "^4.17.13",
+    "@types/jest": "^29.5.14",
+    "@types/jsonwebtoken": "^9.0.1",
+    "@types/lodash": "^4.14.200",
+    "@types/ms": "^0.7.34",
+    "@types/multer": "^1.4.7",
+    "@types/node": "^24.2.0",
+    "@types/node-fetch": "^2.6.11",
+    "@types/node-jose": "^1.1.12",
+    "@types/tiny-async-pool": "^2.0.0",
+    "@types/uuid": "^8.3.4",
+    "@typescript-eslint/eslint-plugin": "^5.13.0",
+    "@typescript-eslint/parser": "^8.39.0",
+    "cors": "^2.8.5",
+    "cspell": "^5.18.5",
+    "eslint": "^8.10.0",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jest": "^26.1.1",
+    "express": "^4.17.3",
+    "jest": "^29.7.0",
+    "multer": "^1.4.5-lts.1",
+    "node-fetch": "^2.7.0",
+    "npm-run-all": "^4.1.5",
+    "postgres": "^3.4.5",
+    "rollup-plugin-commonjs": "^10.1.0",
+    "rollup-plugin-typescript2": "^0.31.2",
+    "ts-jest": "^29.4.1",
+    "typescript": "5.9.2",
+    "yargs": "^17.5.1",
+    "zod": "^4.0.15"
+  },
+  "exports": {
+    ".": {
+      "browser": "./dist/cjs/index.js",
+      "import": "./dist/esm/index.js",
+      "require": "./dist/cjs/index.js"
+    },
+    "./*": {
+      "browser": "./dist/cjs/*.js",
+      "import": "./dist/esm/*.js",
+      "require": "./dist/cjs/*.js"
+    },
+    "./assertions": {
+      "browser": "./dist/cjs/assertions/index.js",
+      "import": "./dist/esm/assertions/index.js",
+      "require": "./dist/cjs/assertions/index.js"
+    },
+    "./aws": {
+      "browser": "./dist/cjs/aws/index.js",
+      "import": "./dist/esm/aws/index.js",
+      "require": "./dist/cjs/aws/index.js"
+    },
+    "./config": {
+      "browser": "./dist/cjs/config/index.js",
+      "import": "./dist/esm/config/index.js",
+      "require": "./dist/cjs/config/index.js"
+    },
+    "./errors": {
+      "browser": "./dist/cjs/errors/index.js",
+      "import": "./dist/esm/errors/index.js",
+      "require": "./dist/cjs/errors/index.js"
+    },
+    "./fetch": {
+      "browser": "./dist/cjs/fetch/index.js",
+      "import": "./dist/esm/fetch/index.js",
+      "require": "./dist/cjs/fetch/index.js"
+    },
+    "./guards": {
+      "browser": "./dist/cjs/guards/index.js",
+      "import": "./dist/esm/guards/index.js",
+      "require": "./dist/cjs/guards/index.js"
+    },
+    "./middleware": {
+      "browser": "./dist/cjs/middleware/index.js",
+      "import": "./dist/esm/middleware/index.js",
+      "require": "./dist/cjs/middleware/index.js"
+    },
+    "./misc": {
+      "browser": "./dist/cjs/misc/index.js",
+      "import": "./dist/esm/misc/index.js",
+      "require": "./dist/cjs/misc/index.js"
+    },
+    "./pagination": {
+      "browser": "./dist/cjs/pagination/index.js",
+      "import": "./dist/esm/pagination/index.js",
+      "require": "./dist/cjs/pagination/index.js"
+    },
+    "./routing": {
+      "browser": "./dist/cjs/routing/index.js",
+      "import": "./dist/esm/routing/index.js",
+      "require": "./dist/cjs/routing/index.js"
+    },
+    "./services": {
+      "browser": "./dist/cjs/services/index.js",
+      "import": "./dist/esm/services/index.js",
+      "require": "./dist/cjs/services/index.js"
+    },
+    "./services/launchParams": {
+      "browser": "./dist/cjs/services/launchParams/index.js",
+      "import": "./dist/esm/services/launchParams/index.js",
+      "require": "./dist/cjs/services/launchParams/index.js"
+    },
+    "./services/exercisesGateway": {
+      "browser": "./dist/cjs/services/exercisesGateway/index.js",
+      "import": "./dist/esm/services/exercisesGateway/index.js",
+      "require": "./dist/cjs/services/exercisesGateway/index.js"
+    },
+    "./services/accountsGateway": {
+      "browser": "./dist/cjs/services/accountsGateway/index.js",
+      "import": "./dist/esm/services/accountsGateway/index.js",
+      "require": "./dist/cjs/services/accountsGateway/index.js"
+    },
+    "./services/logger": {
+      "browser": "./dist/cjs/services/logger/index.js",
+      "import": "./dist/esm/services/logger/index.js",
+      "require": "./dist/cjs/services/logger/index.js"
+    },
+    "./services/searchProvider": {
+      "browser": "./dist/cjs/services/searchProvider/index.js",
+      "import": "./dist/esm/services/searchProvider/index.js",
+      "require": "./dist/cjs/services/searchProvider/index.js"
+    },
+    "./services/documentStore": {
+      "browser": "./dist/cjs/services/documentStore/index.js",
+      "import": "./dist/esm/services/documentStore/index.js",
+      "require": "./dist/cjs/services/documentStore/index.js"
+    },
+    "./services/documentStore/unversioned": {
+      "browser": "./dist/cjs/services/documentStore/unversioned/index.js",
+      "import": "./dist/esm/services/documentStore/unversioned/index.js",
+      "require": "./dist/cjs/services/documentStore/unversioned/index.js"
+    },
+    "./services/documentStore/versioned": {
+      "browser": "./dist/cjs/services/documentStore/versioned/index.js",
+      "import": "./dist/esm/services/documentStore/versioned/index.js",
+      "require": "./dist/cjs/services/documentStore/versioned/index.js"
+    },
+    "./services/lrsGateway": {
+      "browser": "./dist/cjs/services/lrsGateway/index.js",
+      "import": "./dist/esm/services/lrsGateway/index.js",
+      "require": "./dist/cjs/services/lrsGateway/index.js"
+    },
+    "./services/authProvider": {
+      "browser": "./dist/cjs/services/authProvider/index.js",
+      "import": "./dist/esm/services/authProvider/index.js",
+      "require": "./dist/cjs/services/authProvider/index.js"
+    },
+    "./services/authProvider/utils": {
+      "browser": "./dist/cjs/services/authProvider/utils/index.js",
+      "import": "./dist/esm/services/authProvider/utils/index.js",
+      "require": "./dist/cjs/services/authProvider/utils/index.js"
+    },
+    "./services/apiGateway": {
+      "browser": "./dist/cjs/services/apiGateway/index.js",
+      "import": "./dist/esm/services/apiGateway/index.js",
+      "require": "./dist/cjs/services/apiGateway/index.js"
+    },
+    "./services/fileServer": {
+      "browser": "./dist/cjs/services/fileServer/index.js",
+      "import": "./dist/esm/services/fileServer/index.js",
+      "require": "./dist/cjs/services/fileServer/index.js"
+    },
+    "./services/postgresConnection": {
+      "browser": "./dist/cjs/services/postgresConnection/index.js",
+      "import": "./dist/esm/services/postgresConnection/index.js",
+      "require": "./dist/cjs/services/postgresConnection/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "script"
+  ],
+  "license": "MIT",
+  "main": "index.js",
+  "peerDependencies": {
+    "@opensearch-project/opensearch": "^1.2.0",
+    "cors": "^2.8.5",
+    "express": "^4.17.3",
+    "multer": "^1.4.5-lts.1",
+    "postgres": "^3.4.5",
+    "zod": "^4.0.15"
+  },
+  "scripts": {
+    "build": "./script/build.bash",
+    "build:clean": "yarn clean && yarn build",
+    "ci": "CI=true npm-run-all ci:*",
+    "ci:lint": "eslint --max-warnings=0 .",
+    "ci:spelling": "git ls-files | yarn cspell -c ../../cspell.json --file-list stdin",
+    "ci:test": "yarn jest --coverage",
+    "ci:typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist",
+    "coverage-report": "open coverage/index.html",
+    "start": "./script/build.bash --watch",
+    "test": "yarn jest"
+  },
+  "sideEffects": false,
+  "types": "index.d.ts",
+  "typesVersions": {
+    "*": {
+      "*": [
+        "dist/esm/*",
+        "dist/esm/*.d.ts",
+        "dist/esm/*/index.d.ts"
+      ]
+    }
+  }
+}