only_ever_generator 8.4.6 → 8.4.8

package/src/gap_fill/calculate_gap_fill.ts

@@ -1,50 +1,50 @@
-function isEmpty(obj: object): boolean {
-  return Object.keys(obj).length === 0;
-}
-
-export function gapFilling(typologyResponse: any, cardgenResponse: any) {
-  let allConcepts: string[] = [];
-  let allFacts: string[] = [];
-  let generatedConceptsList: string[] = [];
-  let generatedFactsList: string[] = [];
-  let remainingConcepts: string[] = [];
-  let remainingFacts: string[] = [];
-  if (!isEmpty(typologyResponse)) {
-    allConcepts.push(...(typologyResponse.concepts ?? []));
-    allFacts.push(...(typologyResponse?.facts ?? []));
-  }
-
-  if (!isEmpty(cardgenResponse)) {
-    allConcepts.push(...(cardgenResponse.missing_concepts ?? []));
-    allFacts.push(...(cardgenResponse.missing_facts ?? []));
-  }
-
-  if (
-    cardgenResponse.cards_data !== undefined &&
-    cardgenResponse.cards_data?.length != 0
-  ) {
-    for (let card of cardgenResponse.cards_data) {
-      if (card.concepts.length != 0) {
-        generatedConceptsList.push(...card.concepts);
-      }
-      if (card.facts.length != 0) {
-        generatedFactsList.push(...card.facts);
-      }
-    }
-
-    let generatedConceptsSet = Array.from(new Set(generatedConceptsList));
-    let generatedFactsSet = Array.from(new Set(generatedFactsList));
-
-    remainingConcepts = allConcepts.filter(
-      (item) => !generatedConceptsSet.includes(item)
-    );
-    remainingFacts = allFacts.filter(
-      (item) => !generatedFactsSet.includes(item)
-    );
-  }
-
-  return {
-    remainingConcepts: remainingConcepts,
-    remainingFacts: remainingFacts,
-  };
-}
+function isEmpty(obj: object): boolean {
+  return Object.keys(obj).length === 0;
+}
+
+export function gapFilling(typologyResponse: any, cardgenResponse: any) {
+  let allConcepts: string[] = [];
+  let allFacts: string[] = [];
+  let generatedConceptsList: string[] = [];
+  let generatedFactsList: string[] = [];
+  let remainingConcepts: string[] = [];
+  let remainingFacts: string[] = [];
+  if (!isEmpty(typologyResponse)) {
+    allConcepts.push(...(typologyResponse.concepts ?? []));
+    allFacts.push(...(typologyResponse?.facts ?? []));
+  }
+
+  if (!isEmpty(cardgenResponse)) {
+    allConcepts.push(...(cardgenResponse.missing_concepts ?? []));
+    allFacts.push(...(cardgenResponse.missing_facts ?? []));
+  }
+
+  if (
+    cardgenResponse.cards_data !== undefined &&
+    cardgenResponse.cards_data?.length != 0
+  ) {
+    for (let card of cardgenResponse.cards_data) {
+      if (card.concepts.length != 0) {
+        generatedConceptsList.push(...card.concepts);
+      }
+      if (card.facts.length != 0) {
+        generatedFactsList.push(...card.facts);
+      }
+    }
+
+    let generatedConceptsSet = Array.from(new Set(generatedConceptsList));
+    let generatedFactsSet = Array.from(new Set(generatedFactsList));
+
+    remainingConcepts = allConcepts.filter(
+      (item) => !generatedConceptsSet.includes(item)
+    );
+    remainingFacts = allFacts.filter(
+      (item) => !generatedFactsSet.includes(item)
+    );
+  }
+
+  return {
+    remainingConcepts: remainingConcepts,
+    remainingFacts: remainingFacts,
+  };
+}

package/src/helper/get_id_from_title.ts

@@ -1,33 +1,33 @@
-import { setUpMongoClient } from "./mongo_helper";
-
-export const getIdsForTitles = async (titles: string[]) => {
-  try {
-    const database = await setUpMongoClient();
-    const collection = database.collection("_source");
-    const rows = await collection
-      .find({
-        $or: [{ title: { $in: titles } }, { slug: { $in: titles } }],
-      })
-      .project({ _id: 1, title: 1, slug: 1 })
-      .toArray();
-
-    // Create a map of matched titles/slugs to their _ids
-    const matchedMap = new Map();
-    rows.forEach((row: any) => {
-      if (row.title) matchedMap.set(row.title, row._id);
-      if (row.slug) matchedMap.set(row.slug, row._id);
-    });
-
-    // Return result for every title in the input array
-
-    return titles.map((title) => {
-      const id = matchedMap.get(title);
-      return {
-        title,
-        id: id ? id.toString() : null,
-      };
-    });
-  } catch (error) {
-    return [];
-  }
-};
+import { setUpMongoClient } from "./mongo_helper";
+
+export const getIdsForTitles = async (titles: string[]) => {
+  try {
+    const database = await setUpMongoClient();
+    const collection = database.collection("_source");
+    const rows = await collection
+      .find({
+        $or: [{ title: { $in: titles } }, { slug: { $in: titles } }],
+      })
+      .project({ _id: 1, title: 1, slug: 1 })
+      .toArray();
+
+    // Create a map of matched titles/slugs to their _ids
+    const matchedMap = new Map();
+    rows.forEach((row: any) => {
+      if (row.title) matchedMap.set(row.title, row._id);
+      if (row.slug) matchedMap.set(row.slug, row._id);
+    });
+
+    // Return result for every title in the input array
+
+    return titles.map((title) => {
+      const id = matchedMap.get(title);
+      return {
+        title,
+        id: id ? id.toString() : null,
+      };
+    });
+  } catch (error) {
+    return [];
+  }
+};

package/src/helper/mongo_helper.ts

@@ -1,29 +1,29 @@
-import { Db, MongoClient, ObjectId } from "mongodb";
-import config from "../config";
-
-// One client per Lambda container (module scope)
-const localClient = new MongoClient(config.dbUri, {
-  maxPoolSize: 30, // tune (20–50 typical for Lambda)
-  minPoolSize: 0, // don't pin idle sockets
-  maxConnecting: 3, // avoid handshake dogpile
-  waitQueueTimeoutMS: 5000, // backpressure instead of pileups
-  maxIdleTimeMS: 90_000, // drop idle to save ports
-  serverSelectionTimeoutMS: 5000,
-  connectTimeoutMS: 5000,
-  retryWrites: true,
-});
-
-let client: MongoClient;
-async function setUpMongoClient(): Promise<Db> {
-  if (!client) client = localClient;
-
-  return client.db(config.dbName);
-}
-
-// export function setUpMongoClient(dbUrl: string, dbName: string) {
-//   mongoClient = new MongoClient(dbUrl, {});
-//   db = mongoClient.db(dbName);
-//   //   return { mongoClient, db };
-// }
-
-export { setUpMongoClient, ObjectId };
+import { Db, MongoClient, ObjectId } from "mongodb";
+import config from "../config";
+
+// One client per Lambda container (module scope)
+const localClient = new MongoClient(config.dbUri, {
+  maxPoolSize: 30, // tune (20–50 typical for Lambda)
+  minPoolSize: 0, // don't pin idle sockets
+  maxConnecting: 3, // avoid handshake dogpile
+  waitQueueTimeoutMS: 5000, // backpressure instead of pileups
+  maxIdleTimeMS: 90_000, // drop idle to save ports
+  serverSelectionTimeoutMS: 5000,
+  connectTimeoutMS: 5000,
+  retryWrites: true,
+});
+
+let client: MongoClient;
+async function setUpMongoClient(): Promise<Db> {
+  if (!client) client = localClient;
+
+  return client.db(config.dbName);
+}
+
+// export function setUpMongoClient(dbUrl: string, dbName: string) {
+//   mongoClient = new MongoClient(dbUrl, {});
+//   db = mongoClient.db(dbName);
+//   //   return { mongoClient, db };
+// }
+
+export { setUpMongoClient, ObjectId };

package/src/helper/openai_helper.ts

@@ -1,20 +1,20 @@
-import OpenAI from "openai";
-
-export class OpenAIHelper {
-  public openAI: OpenAI;
-  public openAIKey: string;
-  constructor(openAIKey: string) {
-    this.openAI = getOpenAI(openAIKey);
-    this.openAIKey = openAIKey;
-  }
-}
-
-const getOpenAI = (openAIKey: string) => {
-  const openai = new OpenAI({
-    apiKey: openAIKey,
-  });
-
-  return openai;
-};
-
-export { getOpenAI, OpenAI };
+import OpenAI from "openai";
+
+export class OpenAIHelper {
+  public openAI: OpenAI;
+  public openAIKey: string;
+  constructor(openAIKey: string) {
+    this.openAI = getOpenAI(openAIKey);
+    this.openAIKey = openAIKey;
+  }
+}
+
+const getOpenAI = (openAIKey: string) => {
+  const openai = new OpenAI({
+    apiKey: openAIKey,
+  });
+
+  return openai;
+};
+
+export { getOpenAI, OpenAI };

package/src/helper/qdrant_db_methods.ts

@@ -1,77 +1,77 @@
-import qdrantClient from "../services/qdrant_service";
-
-const createCollection = async (collectionName: string) => {
-  await qdrantClient.createCollection(collectionName, {
-    vectors: {
-      size: 1536,
-      distance: "Cosine",
-    },
-  });
-};
-
-const getCollection = async () => {
-  const collection = await qdrantClient.getCollections();
-  return collection;
-};
-
-const getCorrespondingConcepts = async (
-  collectionName: string,
-  embeddings: {
-    text: string;
-    id: string;
-    type: string;
-    reference: string;
-    embedding: number[];
-  }[],
-  threshold: number
-) => {
-  try {
-    const searchQuery = embeddings.map((e) => {
-      return {
-        query: e.embedding,
-        limit: 1,
-        score_threshold: threshold,
-        with_payload: true,
-      };
-    });
-    const results = await qdrantClient.queryBatch(collectionName, {
-      searches: searchQuery,
-    });
-    return results;
-  } catch (error) {
-    console.log(error);
-    throw error;
-  }
-};
-
-const addEmbeddingsToCollection = async (
-  collectionName: string,
-  embeddings: {
-    id: string;
-    vector: number[];
-    payload: {
-      _sources: string[];
-      text: string;
-    };
-  }[]
-) => {
-  const CHUNK_SIZE = 1000;
-  for (let i = 0; i < embeddings.length; i += CHUNK_SIZE) {
-    const batch = embeddings.slice(i, i + CHUNK_SIZE);
-    await qdrantClient.upsert(
-      collectionName,
-
-      {
-        wait: true,
-        points: batch,
-      }
-    );
-  }
-};
-
-export {
-  createCollection,
-  getCollection,
-  addEmbeddingsToCollection,
-  getCorrespondingConcepts as getSimilarConcepts,
-};
+import qdrantClient from "../services/qdrant_service";
+
+const createCollection = async (collectionName: string) => {
+  await qdrantClient.createCollection(collectionName, {
+    vectors: {
+      size: 1536,
+      distance: "Cosine",
+    },
+  });
+};
+
+const getCollection = async () => {
+  const collection = await qdrantClient.getCollections();
+  return collection;
+};
+
+const getCorrespondingConcepts = async (
+  collectionName: string,
+  embeddings: {
+    text: string;
+    id: string;
+    type: string;
+    reference: string;
+    embedding: number[];
+  }[],
+  threshold: number
+) => {
+  try {
+    const searchQuery = embeddings.map((e) => {
+      return {
+        query: e.embedding,
+        limit: 1,
+        score_threshold: threshold,
+        with_payload: true,
+      };
+    });
+    const results = await qdrantClient.queryBatch(collectionName, {
+      searches: searchQuery,
+    });
+    return results;
+  } catch (error) {
+    console.log(error);
+    throw error;
+  }
+};
+
+const addEmbeddingsToCollection = async (
+  collectionName: string,
+  embeddings: {
+    id: string;
+    vector: number[];
+    payload: {
+      _sources: string[];
+      text: string;
+    };
+  }[]
+) => {
+  const CHUNK_SIZE = 1000;
+  for (let i = 0; i < embeddings.length; i += CHUNK_SIZE) {
+    const batch = embeddings.slice(i, i + CHUNK_SIZE);
+    await qdrantClient.upsert(
+      collectionName,
+
+      {
+        wait: true,
+        points: batch,
+      }
+    );
+  }
+};
+
+export {
+  createCollection,
+  getCollection,
+  addEmbeddingsToCollection,
+  getCorrespondingConcepts as getSimilarConcepts,
+};

package/src/helper/schema_helper/build_card_schema.ts

@@ -1,98 +1,74 @@
-import { sanitizeStringsForSchema } from "../../utils/sanitize_strings";
-
-/**
- * Builds card output schema by populating the integrated schemas with concepts, facts, and card-specific schemas
- * @param concepts - Array of concept strings
- * @param facts - Array of fact strings
- * @param card_generation_types - Array of card types to generate (e.g., ["flash", "mcq", "cloze"])
- * @param cardTypeSchemas - Object mapping card types to their integrated schemas from getPrompts (e.g., { flash: {...}, mcq: {...} })
- * @returns Populated schema object ready for use
- */
-export const buildCardSchema = async (
-  concepts: string[],
-  facts: string[],
-  card_generation_types: string[],
-  cardTypeSchemas: Record<string, any>
-) => {
-  if (!cardTypeSchemas || Object.keys(cardTypeSchemas).length === 0) {
-    throw new Error("Card type schemas are required");
-  }
-
-  const conceptsSanitized = sanitizeStringsForSchema(concepts);
-  const factsSanitized = sanitizeStringsForSchema(facts);
-
-  // Use the first available card type schema as the base (all have the same base structure)
-  const firstCardType = card_generation_types.find(
-    (type) => cardTypeSchemas[type]
-  );
-  if (!firstCardType) {
-    throw new Error("No valid card type schema found");
-  }
-
-  // Deep clone to avoid mutating the original
-  const populatedSchema = JSON.parse(
-    JSON.stringify(cardTypeSchemas[firstCardType].schema)
-  );
-
-  // Populate concepts enum
-  if (
-    populatedSchema.schema?.properties?.test_cards?.items?.properties?.concepts
-      ?.items?.enum
-  ) {
-    populatedSchema.schema.properties.test_cards.items.properties.concepts.items.enum =
-      conceptsSanitized;
-  }
-
-  // Populate facts enum
-  if (
-    populatedSchema.schema?.properties?.test_cards?.items?.properties?.facts
-      ?.items?.enum
-  ) {
-    populatedSchema.schema.properties.test_cards.items.properties.facts.items.enum =
-      factsSanitized;
-  }
-
-  // Replace the type enum with the requested card generation types
-  if (
-    populatedSchema.schema?.properties?.test_cards?.items?.properties?.type
-      ?.enum
-  ) {
-    populatedSchema.schema.properties.test_cards.items.properties.type.enum =
-      card_generation_types;
-  }
-
-  // Extract card-specific schemas from the anyOf of each card type schema
-  const filteredCardTypeSchemas: any[] = [];
-  for (const cardType of card_generation_types) {
-    if (cardTypeSchemas[cardType]) {
-      const cardSchema = cardTypeSchemas[cardType];
-      // Extract the card-specific schema from the anyOf array
-      const cardContentSchema =
-        cardSchema.schema?.properties?.test_cards?.items?.properties
-          ?.card_content?.anyOf?.[0];
-      if (cardContentSchema) {
-        filteredCardTypeSchemas.push(cardContentSchema);
-      }
-    }
-  }
-
-  // Replace anyOf array with all card-specific schemas
-  if (filteredCardTypeSchemas.length > 0) {
-    if (
-      populatedSchema.schema?.properties?.test_cards?.items?.properties
-        ?.card_content
-    ) {
-      populatedSchema.schema.properties.test_cards.items.properties.card_content =
-        {
-          anyOf: filteredCardTypeSchemas,
-        };
-    }
-  }
-
-  console.log(
-    "Populated Schema",
-    JSON.stringify(populatedSchema.schema.properties.test_cards.items, null, 2)
-  );
-
-  return populatedSchema;
-};
+import { sanitizeStringsForSchema } from "../../utils/sanitize_strings";
+import { setUpMongoClient, ObjectId } from "../mongo_helper";
+
+// flash:68ef4e1b5a123c94a835c6ab
+// cloze:68ef4eb45a123c94a835c6ac
+// mcq:68ef4f095a123c94a835c6ad
+// match:68ef4fae5a123c94a835c6ae
+export const buildCardSchema = async (
+  concepts: string[],
+  facts: string[],
+  card_generation_types: string[]
+) => {
+  const database = await setUpMongoClient();
+  const objectId = new ObjectId("689c5cbd0d9d3bbda64b9584");
+  const schemaObj = await database.collection("_prompts").findOne({
+    _id: objectId,
+  });
+  if (!schemaObj) {
+    throw new Error("Schema not found");
+  }
+  const schema = schemaObj.schema;
+  if (!schema) {
+    throw new Error("Schema not found");
+  }
+  const conceptsSanitized = sanitizeStringsForSchema(concepts);
+  const factsSanitized = sanitizeStringsForSchema(facts);
+
+  // Fetch individual card type schemas based on card_generation_types
+  const cardTypeObjectIds: { [key: string]: ObjectId } = {
+    flash: new ObjectId("68ef4e1b5a123c94a835c6ab"),
+    cloze: new ObjectId("68ef4eb45a123c94a835c6ac"),
+    mcq: new ObjectId("68ef4f095a123c94a835c6ad"),
+    match: new ObjectId("68ef4fae5a123c94a835c6ae"),
+  };
+
+  const cardTypeSchemas: { [key: string]: any } = {};
+
+  // Only fetch schemas for the requested card types
+  for (const cardType of card_generation_types) {
+    if (cardTypeObjectIds[cardType]) {
+      const cardSchemaObj = await database.collection("_prompts").findOne({
+        _id: cardTypeObjectIds[cardType],
+      });
+      if (cardSchemaObj && cardSchemaObj.schema) {
+        cardTypeSchemas[cardType] = JSON.parse(cardSchemaObj.schema);
+      }
+    }
+  }
+
+  // Deep clone to avoid mutating the original
+  const populatedSchema = JSON.parse(schema);
+  populatedSchema.schema.properties.test_cards.items.properties.concepts.items.enum =
+    conceptsSanitized;
+  populatedSchema.schema.properties.test_cards.items.properties.facts.items.enum =
+    factsSanitized;
+
+  // Replace the type enum with the requested card generation types
+  populatedSchema.schema.properties.test_cards.items.properties.type.enum =
+    card_generation_types;
+
+  // Replace anyOf array with individual schemas based on card type
+  if (Object.keys(cardTypeSchemas).length > 0) {
+    populatedSchema.schema.properties.test_cards.items.properties.card_content =
+      {
+        anyOf: Object.values(cardTypeSchemas),
+      };
+  }
+
+  console.log(
+    "Populated Schema",
+    JSON.stringify(populatedSchema.schema.properties.test_cards.items, null, 2)
+  );
+  return populatedSchema;
+};