only_ever_generator 8.4.6 → 8.4.8

package/src/typology_gen/generate_typology.ts

@@ -1,189 +1,203 @@
-import { OpenAIHelper } from "../helper/openai_helper";
-import { log_error, ParsingError } from "../logger";
-import { promptIds } from "../services/get_prompts";
-import { OpenAiService } from "../services/open_ai_service";
-import { restoreQuotesInString } from "../utils/sanitize_strings";
-
-export class GenerateTypology {
-  public openAiService: OpenAiService;
-  public type: string = "";
-  public sourceId: string;
-  public generationCurriculum: boolean;
-  public content: {
-    title: string;
-    h1_headings?: string[];
-    timecodes?: string[];
-    content: any[];
-  };
-  expectedFields: Array<string>;
-  constructor(
-    openAiService: OpenAiService,
-    type: string,
-    sourceId: string,
-    content: {
-      title: string;
-      h1_headings?: string[];
-      timecodes?: string[];
-      content: any[];
-    },
-    expected_fields: Array<string>,
-    generationCurriculum: boolean
-  ) {
-    this.openAiService = openAiService;
-    this.type = type;
-    this.sourceId = sourceId;
-    this.content = content;
-    this.expectedFields = expected_fields.map((elem: string) =>
-      elem.toString().toUpperCase()
-    );
-    this.generationCurriculum = generationCurriculum;
-  }
-  async generate() {
-    try {
-      const openAIHelper = new OpenAIHelper(this.openAiService.api_key);
-      const openAiResponse: any = await openAIHelper.openAI.responses.create({
-        prompt: {
-          id: promptIds.classify,
-          variables: {
-            heading_type: this.type == "video" ? "timecode" : "h1 heading",
-          },
-          // version: "66",
-        },
-        max_output_tokens: 30000,
-        input: [
-          {
-            role: "user",
-            content: [
-              { type: "input_text", text: JSON.stringify(this.content) },
-            ],
-          },
-        ],
-        store: true,
-      });
-      console.log(openAiResponse);
-      if (!openAiResponse) {
-        await log_error({
-          flow: this.generationCurriculum
-            ? "curriculum_manual_generation"
-            : "manual_generation",
-          data: openAiResponse,
-          error: "empty_openai_response",
-          timestamp: new Date(),
-          source_id: this.sourceId,
-          type: {
-            request_type: "breadth",
-            n: 1,
-          },
-        });
-        return;
-      }
-
-      // Restore quotes in the response (works without original mapping)
-      // if (openAiResponse.output) {
-      //   openAiResponse.output = restoreQuotesInObject(openAiResponse.output);
-      // }
-
-      openAiResponse["request_type"] = {
-        type: "breadth",
-        n: 1,
-      };
-      openAiResponse.metadata = {
-        req_time: openAiResponse.created ?? new Date(),
-        req_type: {
-          type: "breadth",
-          n: 1,
-        },
-        req_tokens: openAiResponse.usage?.input_tokens ?? 0,
-        res_tokens: openAiResponse.usage?.output_tokens ?? 0,
-        prompt: {
-          id: openAiResponse.prompt.id,
-          version: openAiResponse.prompt.version,
-        },
-        // prompt_tokens_details: openAiResponse.usage?.,
-        model: openAiResponse.model,
-        usage: openAiResponse.usage,
-      };
-
-      if (openAiResponse.status == "completed") {
-        console.log("Completed");
-        return this.parseTypologyOnSuccess(openAiResponse);
-      } else {
-        openAiResponse.metadata.err_message = openAiResponse.message;
-        return openAiResponse;
-      }
-    } catch (e: any) {
-      console.log(e);
-      await log_error({
-        flow: this.generationCurriculum
-          ? "curriculum_manual_generation"
-          : "manual_generation",
-        data: e,
-        timestamp: new Date(),
-        error: "error_in_typology_generation",
-        source_id: this.sourceId,
-        type: {
-          request_type: "breadth",
-          n: 1,
-        },
-      });
-    }
-  }
-
-  parseTypologyOnSuccess(responseData: any) {
-    responseData.metadata.status = "completed";
-    const generatedContent = this.parseJson(responseData);
-    console.log(generatedContent.learn_value);
-
-    return {
-      status_code: 200,
-      metadata: [responseData.metadata],
-      field: this.parseFields(generatedContent.field),
-
-      generate_cards: generatedContent.learn_value,
-      summary_cards: generatedContent.summary_cards,
-      // type: responseData.type,
-    };
-  }
-
-  parseJson(response: any) {
-    try {
-      const generatedContent = JSON.parse(response.output_text);
-      const summaryCards = generatedContent?.summary_cards ?? [];
-      if (summaryCards) {
-        const remappedSummaryCards = summaryCards.map((item: any) => {
-          return {
-            ...item,
-            reference: restoreQuotesInString(item.reference),
-          };
-        });
-        return {
-          ...generatedContent,
-          summary_cards: remappedSummaryCards,
-        };
-      }
-      return generatedContent;
-    } catch (e) {
-      throw new ParsingError(
-        "Something went wrong in parsing the json",
-        response
-      );
-    }
-  }
-
-  parseFields(fields: Array<string>) {
-    const fieldKeys = ["primary_field", "secondary_field", "tertiary_field"];
-    return fields.slice(0, 3).map((item, index) => ({
-      [fieldKeys[index]]: item,
-      reconcile: !this.expectedFields.includes(item.toLowerCase()),
-    }));
-  }
-
-  async parseTypologyOnFailure(responseData: any) {
-    responseData.metadata.status = "failed";
-
-    return {
-      status_code: responseData.status_code,
-      metadata: responseData.metadata,
-    };
-  }
-}
+import { OpenAIHelper } from "../helper/openai_helper";
+import { log_error, ParsingError } from "../logger";
+import { OpenAiService } from "../services/open_ai_service";
+import { restoreQuotesInString } from "../utils/sanitize_strings";
+import { jsonrepair } from "jsonrepair";
+
+export class GenerateTypology {
+  public openAiService: OpenAiService;
+  public type: string = "";
+  public sourceId: string;
+  public generationCurriculum: boolean;
+  public content: {
+    title: string;
+    h1_headings?: string[];
+    timecodes?: string[];
+    content: any[];
+  };
+  public promptIdForTypology: string;
+  expectedFields: Array<string>;
+  constructor(
+    openAiService: OpenAiService,
+    type: string,
+    sourceId: string,
+    content: {
+      title: string;
+      h1_headings?: string[];
+      timecodes?: string[];
+      content: any[];
+    },
+    expected_fields: Array<string>,
+    promptIdForTypology: string,
+    generationCurriculum: boolean
+  ) {
+    this.openAiService = openAiService;
+    this.type = type;
+    this.sourceId = sourceId;
+    this.content = content;
+    this.expectedFields = expected_fields.map((elem: string) =>
+      elem.toString().toUpperCase()
+    );
+    this.promptIdForTypology = promptIdForTypology;
+    this.generationCurriculum = generationCurriculum;
+  }
+  async generate() {
+    try {
+      const openAIHelper = new OpenAIHelper(this.openAiService.api_key);
+      const openAiResponse: any = await openAIHelper.openAI.responses.create({
+        prompt: {
+          id: this.promptIdForTypology,
+          variables: {
+            heading_type: this.type == "video" ? "timecode" : "h1 heading",
+          },
+          // version: "66",
+        },
+        max_output_tokens: 30000,
+        input: [
+          {
+            role: "user",
+            content: [
+              { type: "input_text", text: JSON.stringify(this.content) },
+            ],
+          },
+        ],
+        store: true,
+      });
+      console.log(openAiResponse);
+      if (!openAiResponse) {
+        await log_error({
+          flow: this.generationCurriculum
+            ? "curriculum_manual_generation"
+            : "manual_generation",
+          data: openAiResponse,
+          error: "empty_openai_response",
+          timestamp: new Date(),
+          source_id: this.sourceId,
+          type: {
+            request_type: "breadth",
+            n: 1,
+          },
+        });
+        return;
+      }
+
+      // Restore quotes in the response (works without original mapping)
+      // if (openAiResponse.output) {
+      //   openAiResponse.output = restoreQuotesInObject(openAiResponse.output);
+      // }
+
+      openAiResponse["request_type"] = {
+        type: "breadth",
+        n: 1,
+      };
+      openAiResponse.metadata = {
+        req_time: openAiResponse.created ?? new Date(),
+        req_type: {
+          type: "breadth",
+          n: 1,
+        },
+        req_tokens: openAiResponse.usage?.input_tokens ?? 0,
+        res_tokens: openAiResponse.usage?.output_tokens ?? 0,
+        prompt: {
+          id: openAiResponse.prompt.id,
+          version: openAiResponse.prompt.version,
+        },
+        // prompt_tokens_details: openAiResponse.usage?.,
+        model: openAiResponse.model,
+        usage: openAiResponse.usage,
+      };
+
+      if (openAiResponse.status == "completed") {
+        console.log("Completed");
+        return this.parseTypologyOnSuccess(openAiResponse);
+      } else {
+        openAiResponse.metadata.err_message = openAiResponse.message;
+        return openAiResponse;
+      }
+    } catch (e: any) {
+      console.log(e);
+      await log_error({
+        flow: this.generationCurriculum
+          ? "curriculum_manual_generation"
+          : "manual_generation",
+        data: e,
+        timestamp: new Date(),
+        error: "error_in_typology_generation",
+        source_id: this.sourceId,
+        type: {
+          request_type: "breadth",
+          n: 1,
+        },
+      });
+    }
+  }
+
+  parseTypologyOnSuccess(responseData: any) {
+    responseData.metadata.status = "completed";
+    const generatedContent = this.parseJson(responseData);
+    console.log(generatedContent.learn_value);
+
+    return {
+      status_code: 200,
+      metadata: [responseData.metadata],
+      field: this.parseFields(generatedContent.field),
+
+      generate_cards: generatedContent.learn_value,
+      summary_cards: generatedContent.summary_cards,
+      // type: responseData.type,
+    };
+  }
+
+  parseJson(response: any) {
+    let generatedContent: any;
+
+    try {
+      generatedContent = JSON.parse(response.output_text);
+    } catch (e) {
+      // If original JSON parse fails, attempt to repair and parse
+      try {
+        const repairedJson = jsonrepair(response.output_text);
+        generatedContent = JSON.parse(repairedJson);
+      } catch (repairError) {
+        // If repair also fails, throw exception
+        throw new ParsingError(
+          "Something went wrong in repairing the json",
+          response
+        );
+      }
+    }
+
+    // Process the parsed content (same flow for both original and repaired JSON)
+    const summaryCards = generatedContent?.summary_cards ?? [];
+    if (summaryCards) {
+      const remappedSummaryCards = summaryCards.map((item: any) => {
+        return {
+          ...item,
+          reference: restoreQuotesInString(item.reference),
+        };
+      });
+      return {
+        ...generatedContent,
+        summary_cards: remappedSummaryCards,
+      };
+    }
+    return generatedContent;
+  }
+
+  parseFields(fields: Array<string>) {
+    const fieldKeys = ["primary_field", "secondary_field", "tertiary_field"];
+    return fields.slice(0, 3).map((item, index) => ({
+      [fieldKeys[index]]: item,
+      reconcile: !this.expectedFields.includes(item.toLowerCase()),
+    }));
+  }
+
+  async parseTypologyOnFailure(responseData: any) {
+    responseData.metadata.status = "failed";
+
+    return {
+      status_code: responseData.status_code,
+      metadata: responseData.metadata,
+    };
+  }
+}