only_ever_generator 6.0.1 → 6.0.3

package/dist/utils/distributed_quote_restoration.js

@@ -0,0 +1,69 @@
+"use strict";
+/**
+ * Utility for restoring quotes in OpenAI responses across distributed Lambda functions
+ * This works without requiring original context from the API call Lambda
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.restoreQuotesInOpenAiResponse = restoreQuotesInOpenAiResponse;
+exports.restoreQuotesInParsedResponse = restoreQuotesInParsedResponse;
+const sanitize_strings_1 = require("./sanitize_strings");
+/**
+ * Restores quotes in any OpenAI response structure
+ * Can be used in a parsing Lambda that receives the response from an API call Lambda
+ * @param openAiResponse - The complete OpenAI response object
+ * @returns Response with quotes restored in all string fields
+ */
+function restoreQuotesInOpenAiResponse(openAiResponse) {
+    var _a, _b;
+    if (!openAiResponse) {
+        return openAiResponse;
+    }
+    // Create a deep copy to avoid mutating the original
+    const restoredResponse = JSON.parse(JSON.stringify(openAiResponse));
+    // Restore quotes in the main output/content
+    if (restoredResponse.output) {
+        restoredResponse.output = (0, sanitize_strings_1.restoreQuotesInObject)(restoredResponse.output);
+    }
+    // Handle different response formats
+    if (restoredResponse.choices &&
+        ((_b = (_a = restoredResponse.choices[0]) === null || _a === void 0 ? void 0 : _a.message) === null || _b === void 0 ? void 0 : _b.content)) {
+        try {
+            const content = JSON.parse(restoredResponse.choices[0].message.content);
+            restoredResponse.choices[0].message.content = JSON.stringify((0, sanitize_strings_1.restoreQuotesInObject)(content));
+        }
+        catch (error) {
+            // If content is not JSON, restore quotes directly
+            restoredResponse.choices[0].message.content = (0, sanitize_strings_1.restoreQuotesInObject)(restoredResponse.choices[0].message.content);
+        }
+    }
+    // Handle any other potential content fields
+    if (restoredResponse.content) {
+        restoredResponse.content = (0, sanitize_strings_1.restoreQuotesInObject)(restoredResponse.content);
+    }
+    if (restoredResponse.generated_content) {
+        restoredResponse.generated_content = (0, sanitize_strings_1.restoreQuotesInObject)(restoredResponse.generated_content);
+    }
+    return restoredResponse;
+}
+/**
+ * Restores quotes in a parsed response object (after JSON parsing)
+ * Use this when you have already parsed the OpenAI response JSON
+ * @param parsedResponse - The parsed response object
+ * @returns Response with quotes restored
+ */
+function restoreQuotesInParsedResponse(parsedResponse) {
+    return (0, sanitize_strings_1.restoreQuotesInObject)(parsedResponse);
+}
+/**
+ * Example usage in a parsing Lambda function:
+ *
+ * // Receiving response from API call Lambda
+ * const openAiResponse = event.openAiResponse;
+ *
+ * // Restore quotes without needing original mapping
+ * const restoredResponse = restoreQuotesInOpenAiResponse(openAiResponse);
+ *
+ * // Now process the response with original quotes restored
+ * const processedData = processResponse(restoredResponse);
+ */
+//# sourceMappingURL=distributed_quote_restoration.js.map

package/dist/utils/distributed_quote_restoration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"distributed_quote_restoration.js","sourceRoot":"","sources":["../../src/utils/distributed_quote_restoration.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAUH,sEA2CC;AAQD,sEAEC;AA7DD,yDAA2D;AAE3D;;;;;GAKG;AACH,SAAgB,6BAA6B,CAAC,cAAmB;;IAC/D,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,oDAAoD;IACpD,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC;IAEpE,4CAA4C;IAC5C,IAAI,gBAAgB,CAAC,MAAM,EAAE,CAAC;QAC5B,gBAAgB,CAAC,MAAM,GAAG,IAAA,wCAAqB,EAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3E,CAAC;IAED,oCAAoC;IACpC,IACE,gBAAgB,CAAC,OAAO;SACxB,MAAA,MAAA,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,OAAO,0CAAE,OAAO,CAAA,EAC7C,CAAC;QACD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACxE,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAC1D,IAAA,wCAAqB,EAAC,OAAO,CAAC,CAC/B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,kDAAkD;YAClD,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,GAAG,IAAA,wCAAqB,EACjE,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC5C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,4CAA4C;IAC5C,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;QAC7B,gBAAgB,CAAC,OAAO,GAAG,IAAA,wCAAqB,EAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC7E,CAAC;IAED,IAAI,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;QACvC,gBAAgB,CAAC,iBAAiB,GAAG,IAAA,wCAAqB,EACxD,gBAAgB,CAAC,iBAAiB,CACnC,CAAC;IACJ,CAAC;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,SAAgB,6BAA6B,CAAC,cAAmB;IAC/D,OAAO,IAAA,wCAAqB,EAAC,cAAc,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;;;;;;GAWG"}

package/dist/utils/sanitize_strings.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Utility functions for sanitizing strings for OpenAI structured outputs
+ * and restoring them back to their original form without requiring original mapping
+ */
+/**
+ * Sanitizes an array of strings by replacing quotes with a safe placeholder
+ * This approach is stateless and doesn't require storing mappings
+ * @param strings - Array of strings to sanitize
+ * @returns Array of sanitized strings
+ */
+export declare function sanitizeStringsForSchema(strings: string[]): string[];
+/**
+ * Restores quotes in a string by replacing placeholders with actual quotes
+ * This works without needing the original mapping
+ * @param sanitizedString - String with quote placeholders
+ * @returns Original string with quotes restored
+ */
+export declare function restoreQuotesInString(sanitizedString: string): string;
+/**
+ * Recursively restores quotes in an object's string values
+ * This works across Lambda functions without requiring original context
+ * @param obj - Object to process
+ * @returns Object with quotes restored
+ */
+export declare function restoreQuotesInObject(obj: any): any;
+/**
+ * Legacy function signature for backward compatibility
+ * Now ignores the originalMapping parameter since it's not needed
+ * @deprecated Use restoreQuotesInObject(obj) instead
+ */
+export declare function restoreQuotesInObjectLegacy(obj: any, originalMapping?: Map<string, string>): any;
+//# sourceMappingURL=sanitize_strings.d.ts.map

package/dist/utils/sanitize_strings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sanitize_strings.d.ts","sourceRoot":"","sources":["../../src/utils/sanitize_strings.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAEpE;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,eAAe,EAAE,MAAM,GAAG,MAAM,CAErE;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAkBnD;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,GAAG,EAAE,GAAG,EACR,eAAe,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACpC,GAAG,CAEL"}

package/dist/utils/sanitize_strings.js

@@ -0,0 +1,61 @@
+"use strict";
+/**
+ * Utility functions for sanitizing strings for OpenAI structured outputs
+ * and restoring them back to their original form without requiring original mapping
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sanitizeStringsForSchema = sanitizeStringsForSchema;
+exports.restoreQuotesInString = restoreQuotesInString;
+exports.restoreQuotesInObject = restoreQuotesInObject;
+exports.restoreQuotesInObjectLegacy = restoreQuotesInObjectLegacy;
+// Use a unique, reversible encoding for quotes
+const QUOTE_PLACEHOLDER = "___QUOTE___";
+/**
+ * Sanitizes an array of strings by replacing quotes with a safe placeholder
+ * This approach is stateless and doesn't require storing mappings
+ * @param strings - Array of strings to sanitize
+ * @returns Array of sanitized strings
+ */
+function sanitizeStringsForSchema(strings) {
+    return strings.map((str) => str.replace(/"/g, QUOTE_PLACEHOLDER));
+}
+/**
+ * Restores quotes in a string by replacing placeholders with actual quotes
+ * This works without needing the original mapping
+ * @param sanitizedString - String with quote placeholders
+ * @returns Original string with quotes restored
+ */
+function restoreQuotesInString(sanitizedString) {
+    return sanitizedString.replace(new RegExp(QUOTE_PLACEHOLDER, "g"), '"');
+}
+/**
+ * Recursively restores quotes in an object's string values
+ * This works across Lambda functions without requiring original context
+ * @param obj - Object to process
+ * @returns Object with quotes restored
+ */
+function restoreQuotesInObject(obj) {
+    if (typeof obj === "string") {
+        return restoreQuotesInString(obj);
+    }
+    if (Array.isArray(obj)) {
+        return obj.map((item) => restoreQuotesInObject(item));
+    }
+    if (obj && typeof obj === "object") {
+        const result = {};
+        for (const [key, value] of Object.entries(obj)) {
+            result[key] = restoreQuotesInObject(value);
+        }
+        return result;
+    }
+    return obj;
+}
+/**
+ * Legacy function signature for backward compatibility
+ * Now ignores the originalMapping parameter since it's not needed
+ * @deprecated Use restoreQuotesInObject(obj) instead
+ */
+function restoreQuotesInObjectLegacy(obj, originalMapping) {
+    return restoreQuotesInObject(obj);
+}
+//# sourceMappingURL=sanitize_strings.js.map

package/dist/utils/sanitize_strings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sanitize_strings.js","sourceRoot":"","sources":["../../src/utils/sanitize_strings.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAWH,4DAEC;AAQD,sDAEC;AAQD,sDAkBC;AAOD,kEAKC;AA3DD,+CAA+C;AAC/C,MAAM,iBAAiB,GAAG,aAAa,CAAC;AAExC;;;;;GAKG;AACH,SAAgB,wBAAwB,CAAC,OAAiB;IACxD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC;AACpE,CAAC;AAED;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,eAAuB;IAC3D,OAAO,eAAe,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,iBAAiB,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;AAC1E,CAAC;AAED;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,GAAQ;IAC5C,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,OAAO,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAgB,2BAA2B,CACzC,GAAQ,EACR,eAAqC;IAErC,OAAO,qBAAqB,CAAC,GAAG,CAAC,CAAC;AACpC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "only_ever_generator",
-  "version": "6.0.1",
+  "version": "6.0.3",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {

package/src/bootstrap/app.ts

@@ -94,10 +94,6 @@ export class OnlyEverGenerator {
     generate_card: boolean = false
   ): Promise<Array<any>> {
     try {
-      console.log(this.promptIdForTypology);
-      console.log(this.promptIdForConceptFacts);
-      console.log(this.promptIdForCardsGeneration);
-      console.log(this.openAiService.api_key);
       let args = new GenerateArgs(generate_card, generate_typology, false);
       const responseToReturn = [];
       const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
@@ -106,7 +102,6 @@ export class OnlyEverGenerator {
           console.log("Generating typology");
           this.typologyResponse = await this.generateTypology();
           console.log("Typology generated");
-          console.log(this.typologyResponse);
 
           if (
             this.typologyResponse?.status_code == 200 ||

package/src/card_gen/generate_cards.ts

@@ -1,5 +1,8 @@
 import { OpenAIHelper } from "../helper/openai_helper";
-import { cardSchema } from "../helper/schema_helper/build_card_schema";
+import {
+  buildCardSchema,
+  cardSchema,
+} from "../helper/schema_helper/build_card_schema";
 import { ErrorLogger, log_error, ParsingError } from "../logger";
 import { ParseCardResponse } from "../parse/parse_card_response";
 import { OpenAiService } from "../services/open_ai_service";
@@ -43,10 +46,7 @@ export class GenerateCards {
   }
 
   async openAIResponse() {
-    console.log(this.generationVariables);
-    console.log(this.generationVariables.bloom_instructions);
-    console.log(this.generationVariables.card_instructions);
-    console.log(this.generationVariables.card_examples);
+    const schema = await buildCardSchema();
     try {
       let response: any = await this.openAIHelper.openAI.responses.create({
         prompt: {
@@ -71,9 +71,9 @@ export class GenerateCards {
         text: {
           format: {
             type: "json_schema",
-            name: "card_gen_schema",
-            strict: true,
-            schema: cardSchema,
+            name: schema.name,
+            strict: schema.strict,
+            schema: schema.schema,
           },
         },
       });

package/src/helper/schema_helper/build_card_schema.ts

@@ -1,3 +1,23 @@
+import { database, ObjectId, setUp } from "../mongo_helper";
+
+export const buildCardSchema = async () => {
+  setUp();
+  const objectId = new ObjectId("689c5cbd0d9d3bbda64b9584");
+  const schemaObj = await database.collection("_prompts").findOne({
+    _id: objectId,
+  });
+  if (!schemaObj) {
+    throw new Error("Schema not found");
+  }
+  const schema = schemaObj.schema;
+  if (!schema) {
+    throw new Error("Schema not found");
+  }
+  // Deep clone to avoid mutating the original
+  const populatedSchema = JSON.parse(schema);
+  return populatedSchema;
+};
+
 export const cardSchema = {
   type: "object",
   properties: {

package/src/helper/schema_helper/build_classify_summarize_schema.ts

@@ -1,4 +1,5 @@
 import { database, ObjectId, setUp } from "../mongo_helper";
+import { sanitizeStringsForSchema } from "../../utils/sanitize_strings";
 
 // Function to populate enums in a parsed schema
 export const buildClassifySummarizeSchema = async (headings: string[]) => {
@@ -17,13 +18,16 @@ export const buildClassifySummarizeSchema = async (headings: string[]) => {
   // Deep clone to avoid mutating the original
   const populatedSchema = JSON.parse(schema);
 
+  // Sanitize headings to remove quotes for OpenAI structured outputs
+  const sanitizedStrings = sanitizeStringsForSchema(headings);
+
   // Type-safe enum population for reference field in summary_cards
   if (
     populatedSchema.schema?.properties?.summary_cards?.items?.properties
       ?.reference?.enum
   ) {
     populatedSchema.schema.properties.summary_cards.items.properties.reference.enum =
-      headings;
+      sanitizedStrings;
   }
 
   return populatedSchema;

package/src/helper/schema_helper/build_concept_facts_schema.ts

@@ -1,5 +1,6 @@
 import { ObjectId } from "mongodb";
 import { database, setUp } from "../mongo_helper";
+import { sanitizeStringsForSchema } from "../../utils/sanitize_strings";
 
 // Function to populate enums in a parsed schema
 export async function buildConceptFactSchema(
@@ -28,13 +29,16 @@ export async function buildConceptFactSchema(
 
   const allowedRefs = [...headings, ""];
 
+  // Sanitize headings to remove quotes for OpenAI structured outputs
+  const sanitizedStrings = sanitizeStringsForSchema(allowedRefs);
+
   // Type-safe enum population for reference field
   if (
     populatedSchema.schema?.properties?.concepts_facts?.items?.properties
       ?.reference?.enum
   ) {
     populatedSchema.schema.properties.concepts_facts.items.properties.reference.enum =
-      allowedRefs;
+      sanitizedStrings;
   }
 
   return populatedSchema;

package/src/index.ts

@@ -16,68 +16,69 @@ import { BaseParamType } from "./types/base_param_type";
 
 export { OnlyEverGenerator };
 
-// (async () => {
-//   const openAIKey =
-//     "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";
-//   const promptForSummary = {
-//     promptId: "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
-//     version: "13",
-//   };
-//   const promptForConceptFacts = {
-//     promptId: "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
-//     version: "6",
-//   };
+(async () => {
+  const openAIKey =
+    "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";
+  const promptForSummary = {
+    promptId: "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
+    version: "13",
+  };
+  const promptForConceptFacts = {
+    promptId: "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
+    version: "6",
+  };
 
-//   setUp();
+  setUp();
 
-//   const source = await database.collection("_source").findOne({
-//     _id: new ObjectId("6753b1ec56e5e922b57ee2df"),
-//   });
+  const source = await database.collection("_source").findOne({
+    _id: new ObjectId("6753b1968b647549b03cc3fe"),
+  });
 
-//   const contentForGen = {
-//     prompt: {
-//       typology: "",
-//       card_generation: "",
-//     },
-//     content: {
-//       source_id: source?._id?.toString() || "",
-//       title: source?.title || "",
-//       type: source?.source_type || "",
-//       headings: source?.headings || [],
-//       content: source?.content || [],
-//       fields: source?.fields || [],
-//       taxonomy: source?.generate_cards
-//         ? {
-//             generate_cards: source?.generate_cards || {
-//               state: false,
-//               reason: "",
-//             },
-//             concepts_facts: source?.concepts_facts || [],
-//             fields: source?.fields || [],
-//           }
-//         : undefined,
-//     },
-//   };
-//   const prompts = await getPrompts("text");
-//   const bloomInstructions = prompts.bloom_instructions;
-//   const cardInstructions = prompts.card_instructions;
-//   const cardExamples = prompts.card_examples;
+  const contentForGen = {
+    prompt: {
+      typology: "",
+      card_generation: "",
+    },
+    content: {
+      source_id: source?._id?.toString() || "",
+      title: source?.title || "",
+      type: source?.source_type || "",
+      headings: source?.headings || [],
+      content: source?.content || [],
+      fields: source?.fields || [],
+      taxonomy: source?.generate_cards
+        ? {
+            generate_cards: source?.generate_cards || {
+              state: false,
+              reason: "",
+            },
+            concepts_facts: source?.concepts_facts || [],
+            fields: source?.fields || [],
+          }
+        : undefined,
+    },
+  };
+  const prompts = await getPrompts("text");
+  const bloomInstructions = prompts.bloom_instructions;
+  const cardInstructions = prompts.card_instructions;
+  const cardExamples = prompts.card_examples;
 
-//   console.log("Initializing OnlyEverGenerator");
-//   const onlyEverGenerator = new OnlyEverGenerator(
-//     process.env.OPEN_AI_KEY as string,
-//     "o4-mini",
-//     contentForGen,
+  console.log("Initializing OnlyEverGenerator");
+  const onlyEverGenerator = new OnlyEverGenerator(
+    process.env.OPEN_AI_KEY as string,
+    "o4-mini",
+    contentForGen,
 
-//     "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
-//     "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
-//     "pmpt_688118a923e4819098176a13a2f401920d2ea17d881cc6c6",
-//     {
-//       bloom_instructions: bloomInstructions,
-//       card_instructions: cardInstructions,
-//       card_examples: cardExamples,
-//     }
-//   );
-//   const response = await onlyEverGenerator.generate(true, true);
-//   console.log(response);
-// })();
+    "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
+    "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
+    "pmpt_688118a923e4819098176a13a2f401920d2ea17d881cc6c6",
+    {
+      bloom_instructions: bloomInstructions,
+      card_instructions: cardInstructions,
+      card_examples: cardExamples,
+    },
+    false
+  );
+  const response = await onlyEverGenerator.generate(true, true);
+  console.log(response);
+})();

package/src/typology_gen/generate_concept_facts.ts

@@ -2,6 +2,10 @@ import { buildConceptFactSchema } from "../helper/schema_helper/build_concept_fa
 import { OpenAIHelper } from "../helper/openai_helper";
 import { OpenAiService } from "../services/open_ai_service";
 import { ErrorLogger, log_error, ParsingError } from "../logger";
+import {
+  restoreQuotesInObject,
+  restoreQuotesInString,
+} from "../utils/sanitize_strings";
 
 export class GenerateConceptFacts {
   public openAiService: OpenAiService;
@@ -94,6 +98,12 @@ export class GenerateConceptFacts {
         });
         return;
       }
+
+      // // Restore quotes in the response (works without original mapping)
+      // if (openAiResponse.output) {
+      //   openAiResponse.output = restoreQuotesInObject(openAiResponse.output);
+      // }
+
       openAiResponse.metadata = {
         req_time: openAiResponse.created ?? new Date(),
         req_type: {
@@ -136,7 +146,14 @@ export class GenerateConceptFacts {
   parseJson(response: any) {
     try {
       const concepts_facts = JSON.parse(response.output_text).concepts_facts;
-      return concepts_facts;
+      if (concepts_facts) {
+        return concepts_facts.map((item: any) => {
+          return {
+            ...item,
+            reference: restoreQuotesInString(item.reference),
+          };
+        });
+      }
     } catch (e) {
       throw new ParsingError(
         "Something went wrong in parsing the json",

package/src/typology_gen/generate_typology.ts

@@ -4,6 +4,10 @@ import { ErrorLogger, log_error, ParsingError } from "../logger";
 import { OpenAiService } from "../services/open_ai_service";
 import { text } from "stream/consumers";
 import { buildClassifySummarizeSchema } from "../helper/schema_helper/build_classify_summarize_schema";
+import {
+  restoreQuotesInObject,
+  restoreQuotesInString,
+} from "../utils/sanitize_strings";
 
 export class GenerateTypology {
   public openAiService: OpenAiService;
@@ -97,6 +101,12 @@ export class GenerateTypology {
         });
         return;
       }
+
+      // Restore quotes in the response (works without original mapping)
+      // if (openAiResponse.output) {
+      //   openAiResponse.output = restoreQuotesInObject(openAiResponse.output);
+      // }
+
       openAiResponse["request_type"] = {
         type: "breadth",
         n: 1,
@@ -158,6 +168,19 @@ export class GenerateTypology {
   parseJson(response: any) {
     try {
       const generatedContent = JSON.parse(response.output_text);
+      const summaryCards = generatedContent?.summary_cards ?? [];
+      if (summaryCards) {
+        const remappedSummaryCards = summaryCards.map((item: any) => {
+          return {
+            ...item,
+            reference: restoreQuotesInString(item.reference),
+          };
+        });
+        return {
+          ...generatedContent,
+          summary_cards: remappedSummaryCards,
+        };
+      }
       return generatedContent;
     } catch (e) {
       throw new ParsingError(

package/src/utils/distributed_quote_restoration.ts

@@ -0,0 +1,80 @@
+/**
+ * Utility for restoring quotes in OpenAI responses across distributed Lambda functions
+ * This works without requiring original context from the API call Lambda
+ */
+
+import { restoreQuotesInObject } from "./sanitize_strings";
+
+/**
+ * Restores quotes in any OpenAI response structure
+ * Can be used in a parsing Lambda that receives the response from an API call Lambda
+ * @param openAiResponse - The complete OpenAI response object
+ * @returns Response with quotes restored in all string fields
+ */
+export function restoreQuotesInOpenAiResponse(openAiResponse: any): any {
+  if (!openAiResponse) {
+    return openAiResponse;
+  }
+
+  // Create a deep copy to avoid mutating the original
+  const restoredResponse = JSON.parse(JSON.stringify(openAiResponse));
+
+  // Restore quotes in the main output/content
+  if (restoredResponse.output) {
+    restoredResponse.output = restoreQuotesInObject(restoredResponse.output);
+  }
+
+  // Handle different response formats
+  if (
+    restoredResponse.choices &&
+    restoredResponse.choices[0]?.message?.content
+  ) {
+    try {
+      const content = JSON.parse(restoredResponse.choices[0].message.content);
+      restoredResponse.choices[0].message.content = JSON.stringify(
+        restoreQuotesInObject(content)
+      );
+    } catch (error) {
+      // If content is not JSON, restore quotes directly
+      restoredResponse.choices[0].message.content = restoreQuotesInObject(
+        restoredResponse.choices[0].message.content
+      );
+    }
+  }
+
+  // Handle any other potential content fields
+  if (restoredResponse.content) {
+    restoredResponse.content = restoreQuotesInObject(restoredResponse.content);
+  }
+
+  if (restoredResponse.generated_content) {
+    restoredResponse.generated_content = restoreQuotesInObject(
+      restoredResponse.generated_content
+    );
+  }
+
+  return restoredResponse;
+}
+
+/**
+ * Restores quotes in a parsed response object (after JSON parsing)
+ * Use this when you have already parsed the OpenAI response JSON
+ * @param parsedResponse - The parsed response object
+ * @returns Response with quotes restored
+ */
+export function restoreQuotesInParsedResponse(parsedResponse: any): any {
+  return restoreQuotesInObject(parsedResponse);
+}
+
+/**
+ * Example usage in a parsing Lambda function:
+ *
+ * // Receiving response from API call Lambda
+ * const openAiResponse = event.openAiResponse;
+ *
+ * // Restore quotes without needing original mapping
+ * const restoredResponse = restoreQuotesInOpenAiResponse(openAiResponse);
+ *
+ * // Now process the response with original quotes restored
+ * const processedData = processResponse(restoredResponse);
+ */