only_ever_generator 8.4.6 → 8.4.8

package/src/typology_gen/summarize.ts

@@ -1,164 +1,176 @@
-import { OpenAIHelper } from "../helper/openai_helper";
-import { OpenAiService } from "../services/open_ai_service";
-import { log_error, ParsingError } from "../logger";
-import { restoreQuotesInString } from "../utils/sanitize_strings";
-import { buildSummarySchema } from "../helper/schema_helper/build_summary_schema";
-import { GenerationVariablesSchema } from "../types/generation_variables_schema";
-import { promptIds } from "../services/get_prompts";
-
-export class GenerateSummaryCards {
-  public openAiService: OpenAiService;
-  public openAIHelper: OpenAIHelper;
-  public sourceId: string;
-  public generationCurriculum: boolean;
-  public generationVariablesSchema: GenerationVariablesSchema;
-  public content: {
-    title: string;
-    h1_headings?: string[];
-    timecodes?: string[];
-    content: any[];
-  };
-  public type: string = "";
-  constructor(
-    openAiService: OpenAiService,
-    sourceId: string,
-    content: {
-      title: string;
-      h1_headings?: string[];
-      timecodes?: string[];
-      content: any[];
-    },
-    type: string,
-    generationCurriculum: boolean,
-    generationVariablesSchema: GenerationVariablesSchema
-  ) {
-    this.openAiService = openAiService;
-    this.openAIHelper = new OpenAIHelper(this.openAiService.api_key);
-    this.content = content;
-    this.type = type;
-    this.sourceId = sourceId;
-    this.generationCurriculum = generationCurriculum;
-    this.generationVariablesSchema = generationVariablesSchema;
-  }
-
-  async generate() {
-    try {
-      const headings =
-        this.type === "text"
-          ? this.content.h1_headings || [""]
-          : this.content.timecodes || [""];
-      const schema = await buildSummarySchema(
-        headings.length > 0 ? headings : [""],
-        this.generationVariablesSchema.output_schema.summary
-      );
-      // const schema = this.generationVariablesSchema.output_schema.summary;
-      const openAiResponse: any =
-        await this.openAIHelper.openAI.responses.create({
-          prompt: {
-            id: promptIds.summarize,
-            variables: {
-              heading_type: this.type == "video" ? "timecode" : "h1 heading",
-            },
-          },
-          max_output_tokens: 30000,
-          input: [
-            {
-              role: "user",
-              content: [
-                {
-                  type: "input_text",
-                  text: JSON.stringify(this.content),
-                },
-              ],
-            },
-          ],
-          store: true,
-          text: {
-            format: {
-              type: "json_schema",
-              name: schema.name,
-              strict: schema.strict,
-              schema: schema.schema,
-            },
-          },
-        });
-      if (!openAiResponse) {
-        await log_error({
-          flow: "manual_generation",
-          data: openAiResponse,
-          error: "empty_openai_response",
-          timestamp: new Date(),
-          source_id: this.sourceId,
-          type: {
-            request_type: "summary_cards",
-            n: 1,
-          },
-        });
-        return;
-      }
-
-      openAiResponse.metadata = {
-        req_time: openAiResponse.created ?? new Date(),
-        req_type: {
-          type: "breadth",
-          sub_type: "summary_cards",
-          n: 1,
-        },
-        req_tokens: openAiResponse.usage?.input_tokens ?? 0,
-        res_tokens: openAiResponse.usage?.output_tokens ?? 0,
-        prompt: {
-          id: openAiResponse.prompt.id,
-          version: openAiResponse.prompt.version,
-        },
-        // prompt_tokens_details: openAiResponse.usage?.,
-        model: this.openAiService.model,
-        usage: openAiResponse.usage,
-        status: "completed",
-      };
-      const summaryCards = this.parseJson(openAiResponse);
-
-      return {
-        summary_cards: summaryCards,
-        metadata: openAiResponse.metadata,
-      };
-    } catch (error) {
-      console.log(error);
-      await log_error({
-        flow: this.generationCurriculum
-          ? "curriculum_manual_generation"
-          : "manual_generation",
-        data: error,
-        timestamp: new Date(),
-        error: "error_in_summary_cards_generation",
-        source_id: this.sourceId,
-        type: {
-          request_type: "summary_cards",
-          n: 1,
-        },
-      });
-      return {
-        summary_cards: [],
-        metadata: {},
-      };
-    }
-  }
-
-  parseJson(response: any) {
-    try {
-      const summaryCards = JSON.parse(response.output_text).summary_cards;
-      if (summaryCards) {
-        return summaryCards.map((item: any) => {
-          return {
-            ...item,
-            reference: restoreQuotesInString(item.reference),
-          };
-        });
-      }
-    } catch (e) {
-      throw new ParsingError(
-        "Something went wrong in parsing the json",
-        response
-      );
-    }
-  }
-}
+import { OpenAIHelper } from "../helper/openai_helper";
+import { OpenAiService } from "../services/open_ai_service";
+import { log_error, ParsingError } from "../logger";
+import { restoreQuotesInString } from "../utils/sanitize_strings";
+import { buildSummarySchema } from "../helper/schema_helper/build_summary_schema";
+import { jsonrepair } from "jsonrepair";
+
+export class GenerateSummaryCards {
+  public openAiService: OpenAiService;
+  public openAIHelper: OpenAIHelper;
+  public sourceId: string;
+  public generationCurriculum: boolean;
+  public content: {
+    title: string;
+    h1_headings?: string[];
+    timecodes?: string[];
+    content: any[];
+  };
+  public type: string = "";
+  public promptIdForSummaryCards: string;
+  constructor(
+    openAiService: OpenAiService,
+    sourceId: string,
+    content: {
+      title: string;
+      h1_headings?: string[];
+      timecodes?: string[];
+      content: any[];
+    },
+    type: string,
+    promptIdForSummaryCards: string,
+    generationCurriculum: boolean
+  ) {
+    this.openAiService = openAiService;
+    this.openAIHelper = new OpenAIHelper(this.openAiService.api_key);
+    this.content = content;
+    this.promptIdForSummaryCards = promptIdForSummaryCards;
+    this.type = type;
+    this.sourceId = sourceId;
+    this.generationCurriculum = generationCurriculum;
+  }
+
+  async generate() {
+    try {
+      const headings =
+        this.type === "text"
+          ? this.content.h1_headings || [""]
+          : this.content.timecodes || [""];
+      const schema = await buildSummarySchema(
+        headings.length > 0 ? headings : [""],
+        "summary_cards",
+        true
+      );
+      const openAiResponse: any =
+        await this.openAIHelper.openAI.responses.create({
+          prompt: {
+            id: this.promptIdForSummaryCards,
+            variables: {
+              heading_type: this.type == "video" ? "timecode" : "h1 heading",
+            },
+          },
+          max_output_tokens: 30000,
+          input: [
+            {
+              role: "user",
+              content: [
+                {
+                  type: "input_text",
+                  text: JSON.stringify(this.content),
+                },
+              ],
+            },
+          ],
+          store: true,
+          text: {
+            format: {
+              type: "json_schema",
+              name: schema.name,
+              strict: schema.strict,
+              schema: schema.schema,
+            },
+          },
+        });
+      if (!openAiResponse) {
+        await log_error({
+          flow: "manual_generation",
+          data: openAiResponse,
+          error: "empty_openai_response",
+          timestamp: new Date(),
+          source_id: this.sourceId,
+          type: {
+            request_type: "summary_cards",
+            n: 1,
+          },
+        });
+        return;
+      }
+
+      openAiResponse.metadata = {
+        req_time: openAiResponse.created ?? new Date(),
+        req_type: {
+          type: "breadth",
+          sub_type: "summary_cards",
+          n: 1,
+        },
+        req_tokens: openAiResponse.usage?.input_tokens ?? 0,
+        res_tokens: openAiResponse.usage?.output_tokens ?? 0,
+        prompt: {
+          id: openAiResponse.prompt.id,
+          version: openAiResponse.prompt.version,
+        },
+        // prompt_tokens_details: openAiResponse.usage?.,
+        model: this.openAiService.model,
+        usage: openAiResponse.usage,
+        status: "completed",
+      };
+      const summaryCards = this.parseJson(openAiResponse);
+
+      return {
+        summary_cards: summaryCards,
+        metadata: openAiResponse.metadata,
+      };
+    } catch (error) {
+      console.log(error);
+      await log_error({
+        flow: this.generationCurriculum
+          ? "curriculum_manual_generation"
+          : "manual_generation",
+        data: error,
+        timestamp: new Date(),
+        error: "error_in_summary_cards_generation",
+        source_id: this.sourceId,
+        type: {
+          request_type: "summary_cards",
+          n: 1,
+        },
+      });
+      return {
+        summary_cards: [],
+        metadata: {},
+      };
+    }
+  }
+
+  parseJson(response: any) {
+    let parsedJson: any;
+
+    try {
+      parsedJson = JSON.parse(response.output_text);
+    } catch (e) {
+      // If original JSON parse fails, attempt to repair and parse
+      try {
+        const repairedJson = jsonrepair(response.output_text);
+        parsedJson = JSON.parse(repairedJson);
+      } catch (repairError) {
+        // If repair also fails, throw exception
+        throw new ParsingError(
+          "Something went wrong in repairing the json",
+          response
+        );
+      }
+    }
+
+    // Process the parsed content (same flow for both original and repaired JSON)
+    const summaryCards = parsedJson.summary_cards;
+    if (summaryCards) {
+      return summaryCards.map((item: any) => {
+        return {
+          ...item,
+          reference: restoreQuotesInString(item.reference),
+        };
+      });
+    }
+    return [];
+  }
+}

package/src/utils/distributed_quote_restoration.ts

@@ -1,80 +1,80 @@
-/**
- * Utility for restoring quotes in OpenAI responses across distributed Lambda functions
- * This works without requiring original context from the API call Lambda
- */
-
-import { restoreQuotesInObject } from "./sanitize_strings";
-
-/**
- * Restores quotes in any OpenAI response structure
- * Can be used in a parsing Lambda that receives the response from an API call Lambda
- * @param openAiResponse - The complete OpenAI response object
- * @returns Response with quotes restored in all string fields
- */
-export function restoreQuotesInOpenAiResponse(openAiResponse: any): any {
-  if (!openAiResponse) {
-    return openAiResponse;
-  }
-
-  // Create a deep copy to avoid mutating the original
-  const restoredResponse = JSON.parse(JSON.stringify(openAiResponse));
-
-  // Restore quotes in the main output/content
-  if (restoredResponse.output) {
-    restoredResponse.output = restoreQuotesInObject(restoredResponse.output);
-  }
-
-  // Handle different response formats
-  if (
-    restoredResponse.choices &&
-    restoredResponse.choices[0]?.message?.content
-  ) {
-    try {
-      const content = JSON.parse(restoredResponse.choices[0].message.content);
-      restoredResponse.choices[0].message.content = JSON.stringify(
-        restoreQuotesInObject(content)
-      );
-    } catch (error) {
-      // If content is not JSON, restore quotes directly
-      restoredResponse.choices[0].message.content = restoreQuotesInObject(
-        restoredResponse.choices[0].message.content
-      );
-    }
-  }
-
-  // Handle any other potential content fields
-  if (restoredResponse.content) {
-    restoredResponse.content = restoreQuotesInObject(restoredResponse.content);
-  }
-
-  if (restoredResponse.generated_content) {
-    restoredResponse.generated_content = restoreQuotesInObject(
-      restoredResponse.generated_content
-    );
-  }
-
-  return restoredResponse;
-}
-
-/**
- * Restores quotes in a parsed response object (after JSON parsing)
- * Use this when you have already parsed the OpenAI response JSON
- * @param parsedResponse - The parsed response object
- * @returns Response with quotes restored
- */
-export function restoreQuotesInParsedResponse(parsedResponse: any): any {
-  return restoreQuotesInObject(parsedResponse);
-}
-
-/**
- * Example usage in a parsing Lambda function:
- *
- * // Receiving response from API call Lambda
- * const openAiResponse = event.openAiResponse;
- *
- * // Restore quotes without needing original mapping
- * const restoredResponse = restoreQuotesInOpenAiResponse(openAiResponse);
- *
- * // Now process the response with original quotes restored
- * const processedData = processResponse(restoredResponse);
- */
+/**
+ * Utility for restoring quotes in OpenAI responses across distributed Lambda functions
+ * This works without requiring original context from the API call Lambda
+ */
+
+import { restoreQuotesInObject } from "./sanitize_strings";
+
+/**
+ * Restores quotes in any OpenAI response structure
+ * Can be used in a parsing Lambda that receives the response from an API call Lambda
+ * @param openAiResponse - The complete OpenAI response object
+ * @returns Response with quotes restored in all string fields
+ */
+export function restoreQuotesInOpenAiResponse(openAiResponse: any): any {
+  if (!openAiResponse) {
+    return openAiResponse;
+  }
+
+  // Create a deep copy to avoid mutating the original
+  const restoredResponse = JSON.parse(JSON.stringify(openAiResponse));
+
+  // Restore quotes in the main output/content
+  if (restoredResponse.output) {
+    restoredResponse.output = restoreQuotesInObject(restoredResponse.output);
+  }
+
+  // Handle different response formats
+  if (
+    restoredResponse.choices &&
+    restoredResponse.choices[0]?.message?.content
+  ) {
+    try {
+      const content = JSON.parse(restoredResponse.choices[0].message.content);
+      restoredResponse.choices[0].message.content = JSON.stringify(
+        restoreQuotesInObject(content)
+      );
+    } catch (error) {
+      // If content is not JSON, restore quotes directly
+      restoredResponse.choices[0].message.content = restoreQuotesInObject(
+        restoredResponse.choices[0].message.content
+      );
+    }
+  }
+
+  // Handle any other potential content fields
+  if (restoredResponse.content) {
+    restoredResponse.content = restoreQuotesInObject(restoredResponse.content);
+  }
+
+  if (restoredResponse.generated_content) {
+    restoredResponse.generated_content = restoreQuotesInObject(
+      restoredResponse.generated_content
+    );
+  }
+
+  return restoredResponse;
+}
+
+/**
+ * Restores quotes in a parsed response object (after JSON parsing)
+ * Use this when you have already parsed the OpenAI response JSON
+ * @param parsedResponse - The parsed response object
+ * @returns Response with quotes restored
+ */
+export function restoreQuotesInParsedResponse(parsedResponse: any): any {
+  return restoreQuotesInObject(parsedResponse);
+}
+
+/**
+ * Example usage in a parsing Lambda function:
+ *
+ * // Receiving response from API call Lambda
+ * const openAiResponse = event.openAiResponse;
+ *
+ * // Restore quotes without needing original mapping
+ * const restoredResponse = restoreQuotesInOpenAiResponse(openAiResponse);
+ *
+ * // Now process the response with original quotes restored
+ * const processedData = processResponse(restoredResponse);
+ */

package/src/utils/generate_args.ts

@@ -1,29 +1,29 @@
-export class GenerateArgs {
-  public generate_card: boolean = false;
-  public generate_typology: boolean = false;
-  public generate_summary: boolean = false;
-
-  constructor(
-    generate_card: boolean,
-    generate_typology: boolean,
-    generate_summary: boolean
-  ) {
-    this.generate_card = generate_card;
-    this.generate_typology = generate_typology;
-    this.generate_summary = generate_summary;
-  }
-
-  getWhatNeedsToBeGenerated() {
-    let returnData = [];
-    if (this.generate_typology == true) {
-      returnData.push("generate_tyopology");
-    }
-    if (this.generate_summary == true) {
-      returnData.push("generate_summary");
-    }
-    if (this.generate_card == true) {
-      returnData.push("generate_card");
-    }
-    return returnData;
-  }
-}
+export class GenerateArgs {
+  public generate_card: boolean = false;
+  public generate_typology: boolean = false;
+  public generate_summary: boolean = false;
+
+  constructor(
+    generate_card: boolean,
+    generate_typology: boolean,
+    generate_summary: boolean
+  ) {
+    this.generate_card = generate_card;
+    this.generate_typology = generate_typology;
+    this.generate_summary = generate_summary;
+  }
+
+  getWhatNeedsToBeGenerated() {
+    let returnData = [];
+    if (this.generate_typology == true) {
+      returnData.push("generate_tyopology");
+    }
+    if (this.generate_summary == true) {
+      returnData.push("generate_summary");
+    }
+    if (this.generate_card == true) {
+      returnData.push("generate_card");
+    }
+    return returnData;
+  }
+}

package/src/utils/parse_openai_response.ts

@@ -1,19 +1,19 @@
-export function parseOpenAiSuccessResponse(responseData: any) {
-  let choices = JSON.parse(responseData.choices[0].message.content);
-  let usuage = responseData.usage;
-  let createdTime = responseData.created;
-  return {
-    status_code: 200,
-    usage_data: usuage,
-    generated_content: choices,
-    generated_at: new Date(createdTime * 1000),
-  };
-}
-
-export function parseOpenAiFailureResponse(errorResponse: any) {
-  // let statusCode =
-  return {
-    status_code: errorResponse.status,
-    message: errorResponse.data?.error?.code,
-  };
-}
+export function parseOpenAiSuccessResponse(responseData: any) {
+  let choices = JSON.parse(responseData.choices[0].message.content);
+  let usuage = responseData.usage;
+  let createdTime = responseData.created;
+  return {
+    status_code: 200,
+    usage_data: usuage,
+    generated_content: choices,
+    generated_at: new Date(createdTime * 1000),
+  };
+}
+
+export function parseOpenAiFailureResponse(errorResponse: any) {
+  // let statusCode =
+  return {
+    status_code: errorResponse.status,
+    message: errorResponse.data?.error?.code,
+  };
+}