@openstax/ts-utils 1.33.1 → 1.34.0

package/packages/utils/src/services/lrsGateway/xapiUtils.ts

@@ -0,0 +1,262 @@
+import partition from 'lodash/fp/partition';
+import { roundToPrecision } from '../..';
+import { AccountsGateway, MappedUserInfo } from '../accountsGateway';
+import { AuthProvider } from '../authProvider';
+import { ActivityState, resolveAttemptInfo } from './attempt-utils';
+import { LrsGateway, XapiStatement } from '.';
+
+export interface Grade {
+  activityProgress: 'Initialized' | 'Started' | 'inProgress' | 'Submitted' | 'Completed';
+  comment?: string;
+  gradingProgress: 'FullyGraded' | 'Pending' | 'PendingManual' | 'Failed' | 'NotReady';
+  scoreGiven: number;
+  scoreMaximum: number;
+  submission?: {
+    startedAt?: string;
+    submittedAt?: string;
+  };
+  userId?: string;
+}
+
+export const getRegistrationAttemptInfo = async(lrs: LrsGateway, registration: string, options?: {
+  activity?: string;
+  anyUser?: boolean;
+  currentPreference?: 'latest' | 'oldest';
+  ensureSync?: boolean;
+  user?: string;
+}) => {
+  const { currentPreference, ...xapiOptions } = options ?? {};
+  const allStatements = await lrs.getAllXapiStatements({ ...xapiOptions, registration });
+
+  // Partition statements for each user
+  const statementsPerUser: { [key: string]: XapiStatement[] } = {};
+  allStatements.forEach((statement) => {
+    // If we ever support accounts from different statement.actor.account.homePage, this method may need changes
+    // (unless we can guarantee the statement.actor.account.name are unique)
+    statementsPerUser[statement.actor.account.name] ||= [];
+    statementsPerUser[statement.actor.account.name].push(statement);
+  });
+
+  const result: { [key: string]: ReturnType<typeof resolveAttemptInfo> } = {};
+  for (const [userUuid, userStatements] of Object.entries(statementsPerUser)) {
+    result[userUuid] = resolveAttemptInfo(userStatements, { currentPreference });
+  }
+
+  return result;
+};
+
+// generates a payload that can be sent to the LMS, documentation here:
+// lti: http://www.imsglobal.org/spec/lti-ags/v2p0#score-publish-service
+// ltijs: https://cvmcosta.me/ltijs/#/grading
+// Note: "min" is currently completely ignored
+export const getScoreGrade = (
+  score: { scaled?: number; raw?: number; min?: number; max?: number },
+  options: { maxScore?: number; startedAt?: string; submittedAt?: string; userId?: string }
+): Grade => {
+  const { raw, scaled, max } = score;
+  const { maxScore, startedAt, submittedAt, userId } = options;
+
+  const scoreMaximum = maxScore ?? 100;
+
+  const scoreGiven = raw && max
+    ? scoreMaximum / max * raw
+    : scaled
+      ? scaled * scoreMaximum
+      : 0
+  ;
+
+  const submission: { startedAt?: string; submittedAt?: string } = {};
+  if (startedAt) {
+    submission.startedAt = startedAt;
+  }
+  if (submittedAt) {
+    submission.submittedAt = submittedAt;
+  }
+
+  return {
+    userId,
+    activityProgress: submittedAt ? 'Completed' : 'Started',
+    // canvas assumes that anything that isn't 'FullyGraded' requires manual grading and displays a "needs grading" icon.
+    // if you warp your mind you can consider the portion of the assignment which is completed to be fully graded.
+    gradingProgress: 'FullyGraded',
+    scoreMaximum,
+    scoreGiven: roundToPrecision(scoreGiven, -2),
+    submission,
+  };
+};
+
+export type Progress = {
+  scaled: number;
+  max?: number;
+  raw?: number;
+};
+
+export type GradeAndProgress = {
+  grade: Grade;
+  progress: Progress;
+  name?: string;
+};
+
+export type UserActivityInfo = MappedUserInfo<ActivityState>;
+
+// These methods assign 0's to incomplete activities
+const pickAttemptAndCompleted = (
+  state: ActivityState,
+  gradePreference: 'current' | 'best'
+): { attempt?: XapiStatement; completed?: XapiStatement } => {
+
+  if (gradePreference === 'best' && state.bestCompletedAttempt) {
+    return {
+      attempt: state.bestCompletedAttempt,
+      completed: state.bestCompletedAttemptCompleted,
+    };
+  }
+
+  // return current attempt if gradePreference is current
+  return {
+    attempt: state.currentAttempt,
+    completed: state.currentAttemptCompleted,
+  };
+};
+
+const getCompletedActivityStateGradeAndProgress = ({ name, scoreMaximum, state, userId, gradePreference = 'current' }: {
+  name?: string; scoreMaximum?: number; state: ActivityState; userId?: string; gradePreference?: 'current' | 'best';
+}): GradeAndProgress => {
+  const { attempt, completed } = pickAttemptAndCompleted(state, gradePreference);
+
+  return {
+    grade: getScoreGrade(
+      completed?.result?.score || {},
+      {
+        maxScore: scoreMaximum,
+        startedAt: attempt?.timestamp,
+        submittedAt: completed?.timestamp,
+        userId,
+      }
+    ),
+    progress: {
+      scaled: completed ? 1 : 0,
+    },
+    name,
+  };
+};
+
+export const getCompletedUserInfosGradeAndProgress = (
+  infos: UserActivityInfo[],
+  scoreMaximum?: number,
+  gradePreference?: 'current' | 'best'
+) =>
+  infos.map(
+    ({ data, fullName, platformUserId }) =>
+      getCompletedActivityStateGradeAndProgress({
+        name: fullName,
+        scoreMaximum,
+        userId: platformUserId,
+        state: data,
+        gradePreference,
+      })
+  );
+
+export const getCurrentGrade = async(
+  services: {
+    lrs: LrsGateway;
+    ltiAuthProvider: AuthProvider;
+  },
+  registration: string,
+  options?: {
+    currentPreference?: 'latest' | 'oldest';
+    incompleteAttemptCallback?: (info: ActivityState) => Promise<GradeAndProgress>;
+    name?: string;
+    scoreMaximum?: number;
+    userId?: string;
+    gradePreference?: 'current' | 'best';
+  }): Promise<GradeAndProgress | null> => {
+  const user = await services.ltiAuthProvider.getUser();
+  if (!user) { return null; }
+
+  const userId = options?.userId ?? user.uuid;
+  const {
+    currentPreference,
+    incompleteAttemptCallback,
+    name,
+    scoreMaximum,
+    gradePreference,
+  } = options ?? {};
+
+  const infoPerUser = await getRegistrationAttemptInfo(services.lrs, registration, { currentPreference });
+  const userInfo = infoPerUser[user.uuid];
+
+  if (!userInfo) {
+    return getCompletedActivityStateGradeAndProgress({
+      name,
+      scoreMaximum,
+      state: resolveAttemptInfo([]),
+      userId,
+      gradePreference,
+    });
+  }
+  if (userInfo.currentAttemptCompleted || !incompleteAttemptCallback) {
+    return getCompletedActivityStateGradeAndProgress({
+      name,
+      scoreMaximum,
+      state: userInfo,
+      userId,
+      gradePreference,
+    });
+  }
+
+  return incompleteAttemptCallback(userInfo);
+};
+
+export const getAssignmentGrades = async(
+  services: {
+    accountsGateway: AccountsGateway;
+    lrs: LrsGateway;
+  },
+  assignmentIRI: string,
+  registration: string,
+  options?: {
+    anyUser?: boolean;
+    currentPreference?: 'latest' | 'oldest';
+    incompleteAttemptsCallback?: (mappedInfo: UserActivityInfo[]) => Promise<GradeAndProgress[]>;
+    platformId?: string;
+    scoreMaximum?: number;
+    user?: string;
+    gradePreference?: 'current' | 'best';
+  }
+): Promise<GradeAndProgress[]> => {
+  const {
+    anyUser,
+    currentPreference,
+    incompleteAttemptsCallback,
+    platformId,
+    scoreMaximum,
+    user,
+    gradePreference = 'current',
+  } = options ?? {};
+
+  const infoPerUserUuid = await getRegistrationAttemptInfo(
+    services.lrs, registration, { activity: assignmentIRI, anyUser, currentPreference, user }
+  );
+
+  const mappedInfo = await services.accountsGateway.mapUserUuids(
+    infoPerUserUuid, platformId
+  );
+
+  // If no custom callback, just return graded results based on preference
+  if (!incompleteAttemptsCallback) {
+    return getCompletedUserInfosGradeAndProgress(mappedInfo, scoreMaximum, gradePreference);
+  }
+
+  // Partition based on whether the chosen preference has a completed attempt
+  const [incompleteInfo, completedInfo] = partition(
+    (info: UserActivityInfo) => {
+      const { completed } = pickAttemptAndCompleted(info.data, gradePreference);
+      return completed === undefined;
+    }
+  )(mappedInfo);
+
+  return getCompletedUserInfosGradeAndProgress(completedInfo, scoreMaximum, gradePreference).concat(
+    await incompleteAttemptsCallback(incompleteInfo)
+  );
+};

package/packages/utils/src/services/postgresConnection/index.spec.ts

@@ -0,0 +1,170 @@
+import postgres from 'postgres';
+import { createCoreLogger } from '../logger';
+import { PostgresConnection, postgresConnection } from '.';
+
+jest.mock('postgres');
+
+  
+const postgresSpy = postgres as any as jest.Mock;
+afterEach(() => {
+  jest.clearAllMocks();
+  jest.resetAllMocks();
+});
+
+describe('postgresConnection', () => {
+  const config = {
+    local: {
+      host: 'cool host',
+      readHost: 'cool read host', 
+      port: '3',
+      database: 'cool database',
+      username: 'cool username', 
+      password: 'cool password' 
+    }
+  };
+  let connection: PostgresConnection;
+
+  beforeEach(() => {
+    const logger = createCoreLogger(jest.fn());
+    const services = {logger};
+    connection = postgresConnection({})(config)(services);
+  });
+
+  it('initializes connection', async () => {
+    await connection.db();
+    expect(postgresSpy.mock.calls[0][0]).toEqual({
+      database: 'cool database',
+      host: 'cool host',
+      password: 'cool password',
+      port: 3,
+      transform: expect.anything(),
+      username: 'cool username',
+    });
+  });
+  
+  it('initializes readonly connection', async () => {
+    await connection.dbRead();
+    expect(postgresSpy.mock.calls[0][0]).toEqual({
+      database: 'cool database',
+      host: 'cool read host',
+      password: 'cool password',
+      port: 3,
+      transform: expect.anything(),
+      username: 'cool username',
+    });
+  });
+ 
+  describe('migrations', () => {
+    it('creates migrations table', async () => {
+      const dbSpy = jest.fn((..._: any[]) => []);
+      postgresSpy.mockReturnValue(dbSpy);
+      await connection.migrate([]);
+      expect(dbSpy.mock.calls[0][0]).toMatchInlineSnapshot(`
+[
+  "
+    CREATE TABLE IF NOT EXISTS migrations (
+      id SERIAL PRIMARY KEY,
+      name TEXT NOT NULL,
+      ran_at TIMESTAMP NOT NULL DEFAULT NOW()
+    )
+  ",
+]
+`);
+    });
+
+    it('runs migration and records it', async () => {
+      const dbSpy = jest.fn(() => []);
+      const up = jest.fn();
+      postgresSpy.mockReturnValue(dbSpy);
+      await connection.migrate([{name: 'test', up}]);
+      expect(up).toHaveBeenCalled();
+      expect(dbSpy.mock.calls.slice(2)).toMatchInlineSnapshot(`
+[
+  [
+    [
+      "INSERT INTO migrations (name) VALUES (",
+      ")",
+    ],
+    "test",
+  ],
+]
+`);
+    });
+
+    it('doesn\'t run migration if already run', async () => {
+      const dbSpy = jest.fn((query: string[]) => query[0] === 'SELECT name FROM migrations'
+        ? [{name: 'test'}]
+        : []
+      );
+      const up = jest.fn();
+      postgresSpy.mockReturnValue(dbSpy);
+      await connection.migrate([{name: 'test', up}]);
+      expect(up).not.toHaveBeenCalled();
+      expect(dbSpy.mock.calls.slice(2)).toMatchInlineSnapshot('[]');
+    });
+  });
+
+  describe('down', () => {
+    it('does nothing if no connections are established', async () => {
+      const downSpy = jest.fn();
+      postgresSpy.mockReturnValue({end: downSpy});
+      await connection.down();
+      expect(downSpy).not.toHaveBeenCalled();
+    });
+    
+    it('tears down db connection', async () => {
+      const downSpy = jest.fn();
+      postgresSpy.mockReturnValue({end: downSpy});
+
+      await connection.db();
+
+      expect(postgresSpy).toHaveBeenCalledTimes(1);
+      expect(postgresSpy.mock.calls[0][0]).toEqual(expect.objectContaining({
+        host: 'cool host',
+      }));
+
+      await connection.down();
+    
+      expect(downSpy).toHaveBeenCalledTimes(1);
+    });
+    
+    it('tears down dbRead connection', async () => {
+      const downSpy = jest.fn();
+      postgresSpy.mockReturnValue({end: downSpy});
+
+      await connection.dbRead();
+
+      expect(postgresSpy).toHaveBeenCalledTimes(1);
+      expect(postgresSpy.mock.calls[0][0]).toEqual(expect.objectContaining({
+        host: 'cool read host',
+      }));
+
+      await connection.down();
+    
+      expect(downSpy).toHaveBeenCalledTimes(1);
+    });
+    
+    it('tears down db and dbRead connection', async () => {
+      const downSpy1 = jest.fn();
+      const downSpy2 = jest.fn();
+      postgresSpy.mockReturnValueOnce({end: downSpy1});
+      postgresSpy.mockReturnValueOnce({end: downSpy2});
+
+      await connection.db();
+      await connection.dbRead();
+
+      expect(postgresSpy).toHaveBeenCalledTimes(2);
+      expect(postgresSpy.mock.calls[0][0]).toEqual(expect.objectContaining({
+        host: 'cool host',
+      }));
+      expect(postgresSpy.mock.calls[1][0]).toEqual(expect.objectContaining({
+        host: 'cool read host',
+      }));
+
+      await connection.down();
+    
+      expect(downSpy1).toHaveBeenCalledTimes(1);
+      expect(downSpy2).toHaveBeenCalledTimes(1);
+    });
+  });
+});

package/packages/utils/src/services/postgresConnection/index.ts

@@ -0,0 +1,84 @@
+import postgres, {Sql} from 'postgres';
+import { assertNotNaN } from '../../assertions';
+import { ConfigProviderForConfig, resolveConfigValue } from '../../config';
+import { ifDefined } from '../../guards';
+import { once } from '../../misc/helpers';
+import type { Logger } from '../logger';
+
+type Config = {
+  host: string;
+  readHost: string;
+  port: string;
+  database: string;
+  username: string;
+  password: string;
+};
+interface Initializer<C> {
+  configSpace?: C;
+}
+
+export const postgresConnection = <C extends string = 'local'>(initializer: Initializer<C>) => (configProvider: {[_key in C]: ConfigProviderForConfig<Config>}) => {
+  const config = configProvider[ifDefined(initializer.configSpace, 'local' as C)];
+  const host = once(() => resolveConfigValue(config.host));
+  const readHost = once(() => resolveConfigValue(config.readHost));
+  const connectionOptions = once(async() => ({
+    port: assertNotNaN(parseInt(await resolveConfigValue(config.port), 10), new Error('port must be a number')),
+    database: await resolveConfigValue(config.database),
+    username: await resolveConfigValue(config.username),
+    password: await resolveConfigValue(config.password),
+    transform: {
+      column: {to: postgres.fromCamel, from: postgres.toCamel},
+    },
+  }));
+  const connections: Sql[] = [];
+  const sql = once(async() => {
+    const options = await connectionOptions();
+    const connection = postgres({...options, host: await host()});
+    connections.push(connection);
+    return connection;
+  });
+  const sqlRead = once(async() => {
+    const options = await connectionOptions();
+    const connection = postgres({...options, host: await readHost()});
+    connections.push(connection);
+    return connection;
+  });
+
+  return (services: {logger: Logger}) => ({
+    db: sql,
+    dbRead: sqlRead,
+    migrate: async(migrations: Migration[]) => {
+      await migrate(services, await sql(), migrations);
+    },
+    down: () => Promise.all(connections.map(c => c.end()))
+  });
+};
+
+export type PostgresConnection = ReturnType<ReturnType<ReturnType<typeof postgresConnection>>>;
+
+export interface Migration {
+  name: string;
+  up: (db: Sql) => Promise<void>;
+}
+
+async function migrate(services: {logger: Logger}, db: Sql, migrations: Migration[]) {
+  await db`
+    CREATE TABLE IF NOT EXISTS migrations (
+      id SERIAL PRIMARY KEY,
+      name TEXT NOT NULL,
+      ran_at TIMESTAMP NOT NULL DEFAULT NOW()
+    )
+  `;
+
+  const ranMigrations = await db`SELECT name FROM migrations`;
+
+  for (const migration of migrations) {
+    if (ranMigrations.some(({name}) => name === migration.name)) {
+      continue;
+    }
+
+    services.logger.log(`Running migration ${migration.name}`);
+    await migration.up(db);
+    await db`INSERT INTO migrations (name) VALUES (${migration.name})`;
+  }
+}

package/packages/utils/src/services/searchProvider/README.md

@@ -0,0 +1,3 @@
+# searchProvider
+
+provides searching/filtering for entity data, currently integrates with any of the VersionedDocumentStore drivers to provide **in memory** filtering, so it loads all the documents every time. ElasticSearch support will be added at some point. currently it seems like the dynamo scan cost is likely to be less than the Elastic cluster cost for our immediate use cases.

package/packages/utils/src/services/searchProvider/index.ts

@@ -0,0 +1,59 @@
+
+export type FieldType = string | string[] | number | boolean;
+
+/* sorta migrating towards more raw ES stuff in the filters */
+export type ESFilter = {
+  terms: Record<string, FieldType>;
+} | {
+  term: Record<string, FieldType>;
+} | {
+  exists: { field: string };
+} | {
+  nested: { path: string; query: Filter };
+} | {
+  bool: BoolFilter;
+};
+export type Filter = {
+  key: string;
+  value: FieldType;
+} | ESFilter;
+
+export type BoolFilter = {
+  must?: Filter[];
+  must_not?: Filter[];
+  should?: Filter[];
+  filter?: Filter[];
+  minimum_should_match?: number;
+};
+
+type Field =
+  {key: string; type?: 'text'; weight: number}
+  | {key: string; type: 'keyword'}
+  | {key: string; type: 'boolean'};
+
+export interface IndexOptions<T> {
+  body: T;
+  id: string;
+}
+
+export type FieldMapping = {
+  type: 'keyword' | 'text' | 'boolean';
+} | {
+  type: 'date'; format?: string;
+} | {
+  type?: 'nested' | 'object';
+  properties: Record<string, FieldMapping>;
+};
+
+export type FieldMappings = Record<string, FieldMapping>;
+
+export interface SearchOptions {
+  page?: number;
+  query: string | undefined;
+  fields: Field[];
+  filter?: Filter[];
+  must?: Filter[];
+  must_not?: Filter[];
+  should?: Filter[];
+  sort?: {key: string; order: 'asc' | 'desc'}[];
+}