only_ever_generator 8.4.5 → 8.4.7

package/src/typology_gen/generate_concept_facts.ts

@@ -1,169 +1,180 @@
-import { buildConceptFactSchema } from "../helper/schema_helper/build_concept_facts_schema";
-import { OpenAIHelper } from "../helper/openai_helper";
-import { OpenAiService } from "../services/open_ai_service";
-import { ErrorLogger, log_error, ParsingError } from "../logger";
-import {
-  restoreQuotesInObject,
-  restoreQuotesInString,
-} from "../utils/sanitize_strings";
-import { GenerationVariablesSchema } from "../types/generation_variables_schema";
-import { promptIds } from "../services/get_prompts";
-
-export class GenerateConceptFacts {
-  public openAiService: OpenAiService;
-  public openAIHelper: OpenAIHelper;
-  public sourceId: string;
-  public generationCurriculum: boolean;
-  public generationVariablesSchema: GenerationVariablesSchema;
-  public content: {
-    title: string;
-    h1_headings?: string[];
-    timecodes?: string[];
-    content: any[];
-  };
-  public type: string = "";
-  constructor(
-    openAiService: OpenAiService,
-    sourceId: string,
-    content: {
-      title: string;
-      h1_headings?: string[];
-      timecodes?: string[];
-      content: any[];
-    },
-    type: string,
-    generationCurriculum: boolean,
-    generationVariablesSchema: GenerationVariablesSchema
-  ) {
-    this.openAiService = openAiService;
-    this.openAIHelper = new OpenAIHelper(this.openAiService.api_key);
-    this.content = content;
-    this.type = type;
-    this.sourceId = sourceId;
-    this.generationCurriculum = generationCurriculum;
-    this.generationVariablesSchema = generationVariablesSchema;
-  }
-
-  async generate() {
-    try {
-      const headings =
-        this.type === "text"
-          ? this.content.h1_headings || [""]
-          : this.content.timecodes || [""];
-      // const schema =
-      //   this.generationVariablesSchema.output_schema.concepts_facts;
-      const schema = await buildConceptFactSchema(
-        headings.length > 0 ? headings : [""],
-        this.generationVariablesSchema.output_schema.concepts_facts
-      );
-      const openAiResponse: any =
-        await this.openAIHelper.openAI.responses.create({
-          prompt: {
-            id: promptIds.conceptFacts,
-            // version: "200",
-            variables: {
-              heading_type: this.type == "video" ? "timecode" : "h1 heading",
-            },
-          },
-          max_output_tokens: 30000,
-          input: [
-            {
-              role: "user",
-              content: [
-                {
-                  type: "input_text",
-                  text: JSON.stringify(this.content),
-                },
-              ],
-            },
-          ],
-          // input: JSON.stringify(this.content),
-          store: true,
-          text: {
-            format: {
-              type: "json_schema",
-              name: schema.name,
-              strict: schema.strict,
-              schema: schema.schema,
-            },
-          },
-        });
-      if (!openAiResponse) {
-        await log_error({
-          flow: "manual_generation",
-          data: openAiResponse,
-          error: "empty_openai_response",
-          timestamp: new Date(),
-          source_id: this.sourceId,
-          type: {
-            request_type: "concept_fact",
-            n: 1,
-          },
-        });
-        return;
-      }
-
-      // // Restore quotes in the response (works without original mapping)
-      // if (openAiResponse.output) {
-      //   openAiResponse.output = restoreQuotesInObject(openAiResponse.output);
-      // }
-
-      openAiResponse.metadata = {
-        req_time: openAiResponse.created ?? new Date(),
-        req_type: {
-          type: "breadth",
-          sub_type: "concept_fact",
-          n: 1,
-        },
-        req_tokens: openAiResponse.usage?.input_tokens ?? 0,
-        res_tokens: openAiResponse.usage?.output_tokens ?? 0,
-        prompt: openAiResponse.prompt,
-        // prompt_tokens_details: openAiResponse.usage?.,
-        model: this.openAiService.model,
-        usage: openAiResponse.usage,
-        status: "completed",
-      };
-      const concepts_facts = this.parseJson(openAiResponse);
-
-      return {
-        concepts_facts: concepts_facts,
-        metadata: openAiResponse.metadata,
-      };
-    } catch (error) {
-      console.log(error);
-      await log_error({
-        flow: this.generationCurriculum
-          ? "curriculum_manual_generation"
-          : "manual_generation",
-        data: error,
-        timestamp: new Date(),
-        error: "error_in_concept_facts_generation",
-        source_id: this.sourceId,
-        type: {
-          request_type: "concept_fact",
-          n: 1,
-        },
-      });
-      return;
-    }
-  }
-
-  parseJson(response: any) {
-    try {
-      const concepts_facts = JSON.parse(response.output_text).concepts_facts;
-      if (concepts_facts) {
-        return concepts_facts.map((item: any) => {
-          return {
-            ...item,
-            reference: restoreQuotesInString(item.reference),
-          };
-        });
-      }
-    } catch (e) {
-      throw new ParsingError(
-        "Something went wrong in parsing the json",
-        response
-      );
-    }
-  }
-}
+import { buildConceptFactSchema } from "../helper/schema_helper/build_concept_facts_schema";
+import { OpenAIHelper } from "../helper/openai_helper";
+import { OpenAiService } from "../services/open_ai_service";
+import { ErrorLogger, log_error, ParsingError } from "../logger";
+import {
+  restoreQuotesInObject,
+  restoreQuotesInString,
+} from "../utils/sanitize_strings";
+import { jsonrepair } from "jsonrepair";
+
+export class GenerateConceptFacts {
+  public openAiService: OpenAiService;
+  public openAIHelper: OpenAIHelper;
+  public sourceId: string;
+  public generationCurriculum: boolean;
+  public content: {
+    title: string;
+    h1_headings?: string[];
+    timecodes?: string[];
+    content: any[];
+  };
+  public type: string = "";
+  public promptIdForConceptFacts: string;
+  constructor(
+    openAiService: OpenAiService,
+    sourceId: string,
+    content: {
+      title: string;
+      h1_headings?: string[];
+      timecodes?: string[];
+      content: any[];
+    },
+    type: string,
+    promptIdForConceptFacts: string,
+    generationCurriculum: boolean
+  ) {
+    this.openAiService = openAiService;
+    this.openAIHelper = new OpenAIHelper(this.openAiService.api_key);
+    this.content = content;
+    this.promptIdForConceptFacts = promptIdForConceptFacts;
+    this.type = type;
+    this.sourceId = sourceId;
+    this.generationCurriculum = generationCurriculum;
+  }
+
+  async generate() {
+    try {
+      const headings =
+        this.type === "text"
+          ? this.content.h1_headings || [""]
+          : this.content.timecodes || [""];
+      const schema = await buildConceptFactSchema(
+        headings.length > 0 ? headings : [""],
+        "concept_fact_gen_schema",
+        true
+      );
+      const openAiResponse: any =
+        await this.openAIHelper.openAI.responses.create({
+          prompt: {
+            id: this.promptIdForConceptFacts,
+            version: "200",
+            variables: {
+              heading_type: this.type == "video" ? "timecode" : "h1 heading",
+            },
+          },
+          max_output_tokens: 30000,
+          input: [
+            {
+              role: "user",
+              content: [
+                {
+                  type: "input_text",
+                  text: JSON.stringify(this.content),
+                },
+              ],
+            },
+          ],
+          // input: JSON.stringify(this.content),
+          store: true,
+          text: {
+            format: {
+              type: "json_schema",
+              name: schema.name,
+              strict: schema.strict,
+              schema: schema.schema,
+            },
+          },
+        });
+      if (!openAiResponse) {
+        await log_error({
+          flow: "manual_generation",
+          data: openAiResponse,
+          error: "empty_openai_response",
+          timestamp: new Date(),
+          source_id: this.sourceId,
+          type: {
+            request_type: "concept_fact",
+            n: 1,
+          },
+        });
+        return;
+      }
+
+      // // Restore quotes in the response (works without original mapping)
+      // if (openAiResponse.output) {
+      //   openAiResponse.output = restoreQuotesInObject(openAiResponse.output);
+      // }
+
+      openAiResponse.metadata = {
+        req_time: openAiResponse.created ?? new Date(),
+        req_type: {
+          type: "breadth",
+          sub_type: "concept_fact",
+          n: 1,
+        },
+        req_tokens: openAiResponse.usage?.input_tokens ?? 0,
+        res_tokens: openAiResponse.usage?.output_tokens ?? 0,
+        prompt: openAiResponse.prompt,
+        // prompt_tokens_details: openAiResponse.usage?.,
+        model: this.openAiService.model,
+        usage: openAiResponse.usage,
+        status: "completed",
+      };
+      const concepts_facts = this.parseJson(openAiResponse);
+
+      return {
+        concepts_facts: concepts_facts,
+        metadata: openAiResponse.metadata,
+      };
+    } catch (error) {
+      console.log(error);
+      await log_error({
+        flow: this.generationCurriculum
+          ? "curriculum_manual_generation"
+          : "manual_generation",
+        data: error,
+        timestamp: new Date(),
+        error: "error_in_concept_facts_generation",
+        source_id: this.sourceId,
+        type: {
+          request_type: "concept_fact",
+          n: 1,
+        },
+      });
+      return;
+    }
+  }
+
+  parseJson(response: any) {
+    let parsedJson: any;
+
+    try {
+      parsedJson = JSON.parse(response.output_text);
+    } catch (e) {
+      // If original JSON parse fails, attempt to repair and parse
+      try {
+        const repairedJson = jsonrepair(response.output_text);
+        parsedJson = JSON.parse(repairedJson);
+      } catch (repairError) {
+        // If repair also fails, throw exception
+        throw new ParsingError(
+          "Something went wrong in repairing the json",
+          response
+        );
+      }
+    }
+
+    // Process the parsed content (same flow for both original and repaired JSON)
+    const concepts_facts = parsedJson.concepts_facts;
+    if (concepts_facts) {
+      return concepts_facts.map((item: any) => {
+        return {
+          ...item,
+          reference: restoreQuotesInString(item.reference),
+        };
+      });
+    }
+    return [];
+  }
+}