only_ever_generator 8.0.0 → 8.0.2

package/src/generators/concept_facts_generator.ts

@@ -0,0 +1,175 @@
+/**
+ * Standalone ConceptFactsGenerator module
+ * Extracts concepts and facts from content
+ */
+
+import { buildConceptFactSchema } from "../helper/schema_helper/build_concept_facts_schema";
+import { OpenAIHelper } from "../helper/openai_helper";
+import { OpenAiService } from "../services/open_ai_service";
+import { ErrorLogger, log_error, ParsingError } from "../logger";
+import { restoreQuotesInString } from "../utils/sanitize_strings";
+import {
+  ConceptFactsGeneratorConfig,
+  DEFAULT_CONCEPT_FACTS_GENERATOR_CONFIG,
+} from "../configs/concept_facts_generator_config";
+import {
+  validateOpenAIKey,
+  validateOpenAIModel,
+  validatePromptId,
+  validateContentType,
+} from "../utils/validation";
+
+export class ConceptFactsGenerator {
+  private config: ConceptFactsGeneratorConfig;
+  private openAiService: OpenAiService;
+  private openAIHelper: OpenAIHelper;
+
+  constructor(config: ConceptFactsGeneratorConfig) {
+    this.validateConfig(config);
+    this.config = { ...DEFAULT_CONCEPT_FACTS_GENERATOR_CONFIG, ...config };
+    this.openAiService = new OpenAiService(
+      this.config.apiKey,
+      this.config.model
+    );
+    this.openAIHelper = new OpenAIHelper(this.config.apiKey);
+  }
+
+  /**
+   * Generate concepts and facts from the configured content
+   */
+  async generate(): Promise<any> {
+    try {
+      const headings =
+        this.config.type === "text"
+          ? this.config.content.h1_headings || [""]
+          : this.config.content.timecodes || [""];
+
+      const schema = await buildConceptFactSchema(
+        headings.length > 0 ? headings : [""],
+        "concept_fact_gen_schema",
+        true
+      );
+
+      const openAiResponse: any =
+        await this.openAIHelper.openAI.responses.create({
+          prompt: {
+            id: this.config.promptId,
+            variables: {
+              heading_type:
+                this.config.type == "video" ? "timecode" : "h1 heading",
+            },
+          },
+          max_output_tokens: 30000,
+          input: [
+            {
+              role: "user",
+              content: [
+                {
+                  type: "input_text",
+                  text: JSON.stringify(this.config.content),
+                },
+              ],
+            },
+          ],
+          store: true,
+          text: {
+            format: {
+              type: "json_schema",
+              name: schema.name,
+              strict: schema.strict,
+              schema: schema.schema,
+            },
+          },
+        });
+
+      if (!openAiResponse) {
+        await log_error({
+          flow: this.config.generationCurriculum
+            ? "curriculum_manual_generation"
+            : "manual_generation",
+          data: openAiResponse,
+          error: "empty_openai_response",
+          timestamp: new Date(),
+          source_id: this.config.sourceId || "unknown",
+          type: {
+            request_type: "concept_fact",
+            n: 1,
+          },
+        });
+        return;
+      }
+
+      openAiResponse.metadata = {
+        req_time: openAiResponse.created ?? new Date(),
+        req_type: {
+          type: "breadth",
+          sub_type: "concept_fact",
+          n: 1,
+        },
+        req_tokens: openAiResponse.usage?.input_tokens ?? 0,
+        res_tokens: openAiResponse.usage?.output_tokens ?? 0,
+        prompt: openAiResponse.prompt,
+        model: this.config.model,
+        usage: openAiResponse.usage,
+        status: "completed",
+      };
+
+      const concepts_facts = this.parseJson(openAiResponse);
+
+      return {
+        concepts_facts: concepts_facts,
+        metadata: openAiResponse.metadata,
+      };
+    } catch (error) {
+      console.log(error);
+      await log_error({
+        flow: this.config.generationCurriculum
+          ? "curriculum_manual_generation"
+          : "manual_generation",
+        data: error,
+        timestamp: new Date(),
+        error: "error_in_concept_facts_generation",
+        source_id: this.config.sourceId || "unknown",
+        type: {
+          request_type: "concept_fact",
+          n: 1,
+        },
+      });
+      throw error;
+    }
+  }
+
+  private parseJson(response: any) {
+    try {
+      const concepts_facts = JSON.parse(response.output_text).concepts_facts;
+      if (concepts_facts) {
+        return concepts_facts.map((item: any) => {
+          return {
+            ...item,
+            reference: restoreQuotesInString(item.reference),
+          };
+        });
+      }
+    } catch (e) {
+      throw new ParsingError(
+        "Something went wrong in parsing the json",
+        response
+      );
+    }
+  }
+
+  private validateConfig(config: ConceptFactsGeneratorConfig): void {
+    validateOpenAIKey(config.apiKey);
+    validateOpenAIModel(config.model);
+    validatePromptId(config.promptId, "prompt ID");
+    validateContentType(config.type);
+
+    if (!config.content) {
+      throw new Error("Content is required");
+    }
+
+    if (!config.content.title) {
+      throw new Error("Content title is required");
+    }
+  }
+}

package/src/generators/typology_generator.ts

@@ -0,0 +1,208 @@
+/**
+ * Standalone TypologyGenerator module
+ * Analyzes content and determines learning value and classification
+ */
+
+import { getOpenAI, OpenAIHelper } from "../helper/openai_helper";
+import { ErrorLogger, log_error, ParsingError } from "../logger";
+import { OpenAiService } from "../services/open_ai_service";
+import { buildClassifySummarizeSchema } from "../helper/schema_helper/build_classify_summarize_schema";
+import { restoreQuotesInString } from "../utils/sanitize_strings";
+import {
+  TypologyGeneratorConfig,
+  DEFAULT_TYPOLOGY_GENERATOR_CONFIG,
+} from "../configs/typology_generator_config";
+import {
+  validateOpenAIKey,
+  validateOpenAIModel,
+  validatePromptId,
+  validateStringArray,
+} from "../utils/validation";
+
+export class TypologyGenerator {
+  private config: TypologyGeneratorConfig;
+  private openAiService: OpenAiService;
+  private openAIHelper: OpenAIHelper;
+
+  constructor(config: TypologyGeneratorConfig) {
+    this.validateConfig(config);
+    this.config = { ...DEFAULT_TYPOLOGY_GENERATOR_CONFIG, ...config };
+    this.openAiService = new OpenAiService(
+      this.config.apiKey,
+      this.config.model
+    );
+    this.openAIHelper = new OpenAIHelper(this.config.apiKey);
+  }
+
+  /**
+   * Generate typology analysis for the configured content
+   */
+  async generate(): Promise<any> {
+    try {
+      const headings = this.config.content.h1_headings ||
+        this.config.content.timecodes || [""];
+
+      // Use OpenAI Responses API
+      const classifySummarizeSchema = await buildClassifySummarizeSchema(
+        headings.length > 0 ? headings : [""]
+      );
+
+      const openAiResponse: any =
+        await this.openAIHelper.openAI.responses.create({
+          prompt: {
+            id: this.config.promptId,
+            variables: {
+              heading_type: this.config.content.timecodes
+                ? "timecode"
+                : "h1 heading",
+            },
+          },
+          max_output_tokens: 30000,
+          input: [
+            {
+              role: "user",
+              content: [
+                {
+                  type: "input_text",
+                  text: JSON.stringify(this.config.content),
+                },
+              ],
+            },
+          ],
+          store: true,
+          text: {
+            format: {
+              type: "json_schema",
+              name: classifySummarizeSchema.name,
+              strict: true,
+              schema: classifySummarizeSchema.schema,
+            },
+          },
+        });
+
+      console.log(openAiResponse);
+
+      if (!openAiResponse) {
+        await log_error({
+          flow: this.config.generationCurriculum
+            ? "curriculum_manual_generation"
+            : "manual_generation",
+          data: openAiResponse,
+          error: "empty_openai_response",
+          timestamp: new Date(),
+          source_id: this.config.sourceId || "unknown",
+          type: {
+            request_type: "breadth",
+            n: 1,
+          },
+        });
+        return;
+      }
+
+      openAiResponse["request_type"] = {
+        type: "breadth",
+        n: 1,
+      };
+
+      openAiResponse.metadata = {
+        req_time: openAiResponse.created ?? new Date(),
+        req_type: {
+          type: "breadth",
+          n: 1,
+        },
+        req_tokens: openAiResponse.usage?.input_tokens ?? 0,
+        res_tokens: openAiResponse.usage?.output_tokens ?? 0,
+        prompt: openAiResponse.prompt,
+        model: openAiResponse.model,
+        usage: openAiResponse.usage,
+      };
+
+      if (openAiResponse.status == "completed") {
+        console.log("Completed");
+        return this.parseTypologyOnSuccess(openAiResponse);
+      } else {
+        openAiResponse.metadata.err_message = openAiResponse.message;
+        return openAiResponse;
+      }
+    } catch (e: any) {
+      console.log(e);
+      await log_error({
+        flow: this.config.generationCurriculum
+          ? "curriculum_manual_generation"
+          : "manual_generation",
+        data: e,
+        timestamp: new Date(),
+        error: "error_in_typology_generation",
+        source_id: this.config.sourceId || "unknown",
+        type: {
+          request_type: "breadth",
+          n: 1,
+        },
+      });
+      throw e;
+    }
+  }
+
+  private parseTypologyOnSuccess(responseData: any) {
+    responseData.metadata.status = "completed";
+    const generatedContent = this.parseJson(responseData);
+    console.log(generatedContent.learn_value);
+
+    return {
+      status_code: 200,
+      metadata: [responseData.metadata],
+      field: this.parseFields(generatedContent.field),
+      generate_cards: generatedContent.learn_value,
+      summary_cards: generatedContent.summary_cards,
+    };
+  }
+
+  private parseJson(response: any) {
+    try {
+      const generatedContent = JSON.parse(response.output_text);
+      const summaryCards = generatedContent?.summary_cards ?? [];
+
+      if (summaryCards) {
+        const remappedSummaryCards = summaryCards.map((item: any) => {
+          return {
+            ...item,
+            reference: restoreQuotesInString(item.reference),
+          };
+        });
+        return {
+          ...generatedContent,
+          summary_cards: remappedSummaryCards,
+        };
+      }
+      return generatedContent;
+    } catch (e) {
+      throw new ParsingError(
+        "Something went wrong in parsing the json",
+        response
+      );
+    }
+  }
+
+  private parseFields(fields: Array<string>) {
+    const fieldKeys = ["primary_field", "secondary_field", "tertiary_field"];
+    return fields.slice(0, 3).map((item, index) => ({
+      [fieldKeys[index]]: item,
+      reconcile: !this.config.expectedFields.includes(item.toLowerCase()),
+    }));
+  }
+
+  private validateConfig(config: TypologyGeneratorConfig): void {
+    validateOpenAIKey(config.apiKey);
+    validateOpenAIModel(config.model);
+    validatePromptId(config.promptId, "prompt ID");
+    validateStringArray(config.expectedFields, "expected fields", true);
+
+    if (!config.content) {
+      throw new Error("Content is required");
+    }
+
+    if (!config.content.title) {
+      throw new Error("Content title is required");
+    }
+  }
+}

package/src/index.ts

@@ -1,95 +1,60 @@
-// import { OnlyEverGenerator } from "./bootstrap/app";
-// import { LocalConsolidation } from "./embedding_generation/consolidation/local_consolidation";
-// import { embeddingsResp } from "./typology-parsed-response";
-// import { GlobalConsolidation } from "./embedding_generation/consolidation/global_consolidation";
-// import { ParseEmbeddingResponse } from "./embedding_generation/parse_embedding_response";
-// import { OpenAIHelper } from "./helper/openai_helper";
-// import { GenerateTypology } from "./typology_gen/generate_typology";
-// import { OpenAiService } from "./services/open_ai_service";
-// import { GenerateConceptFacts } from "./typology_gen/generate_concept_facts";
-// import { BaseParamType } from "./types/base_param_type";
-// import express from "express";
-// const app = express();
-// const port = 3000;
-
-import { OnlyEverGenerator } from "./bootstrap/app";
-import { database, ObjectId, setUp } from "./helper/mongo_helper";
-import { getPrompts } from "./services/get_prompts";
-import { BaseParamType } from "./types/base_param_type";
-import { ParseCardResponse } from "./parse/parse_card_response";
-import { RawTestCardResponseType } from "./types/raw_card_response_types/generated_card_response_type";
-
-export { OnlyEverGenerator };
-
-// (async () => {
-//   const openAIKey =
-//     "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";
-//   const promptForSummary = {
-//     promptId: "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
-//     // version: "13",
-//   };
-//   const promptForConceptFacts = {
-//     promptId: "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
-//     // version: "6",
-//   };
-
-//   setUp();
-
-//   const source = await database.collection("_source").findOne({
-//     _id: new ObjectId("68b114cdf23998242ce03235"),
-//   });
-
-//   const contentForGen = {
-//     prompt: {
-//       typology: "",
-//       card_generation: "",
-//     },
-//     content: {
-//       source_id: source?._id?.toString() || "",
-//       title: source?.title || "",
-//       type: source?.source_type || "",
-//       headings: source?.headings || [],
-//       content: source?.content || [],
-//       fields: source?.fields || [],
-//       taxonomy: source?.learn_value
-//         ? {
-//             generate_cards: source?.generate_cards || {
-//               state: false,
-//               reason: "",
-//             },
-//             concepts_facts: source?.concepts_facts || [],
-//             fields: source?.fields || [],
-//             learn_value: source?.learn_value || {
-//               value: 0,
-//               reason: "",
-//               bloom_levels: [],
-//               bloom_levels_reason: "",
-//             },
-//           }
-//         : undefined,
-//     },
-//   };
-//   const prompts = await getPrompts("text");
-//   const bloomInstructions = prompts.bloom_instructions;
-//   const cardInstructions = prompts.card_instructions;
-//   const cardExamples = prompts.card_examples;
-
-//   console.log("Initializing OnlyEverGenerator");
-//   const onlyEverGenerator = new OnlyEverGenerator(
-//     process.env.OPEN_AI_KEY as string,
-//     "o4-mini",
-//     contentForGen,
-
-//     "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
-//     "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
-//     "pmpt_688118a923e4819098176a13a2f401920d2ea17d881cc6c6",
-//     {
-//       bloom_instructions: bloomInstructions,
-//       card_instructions: cardInstructions,
-//       card_examples: cardExamples,
-//     },
-//     false
-//   );
-//   const response = await onlyEverGenerator.generate(true, false);
-//   console.log(response);
-// })();
+// Core Generation Modules
+export { CardGenerator } from "./generators/card_generator";
+export { TypologyGenerator } from "./generators/typology_generator";
+export { ConceptFactsGenerator } from "./generators/concept_facts_generator";
+
+// Embedding & Consolidation Modules
+export { EmbeddingGenerator } from "./embeddings/embedding_generator";
+export { LocalConsolidator } from "./consolidation/local_consolidator";
+export { GlobalConsolidator } from "./consolidation/global_consolidator";
+
+// Parser Modules
+export { ContentParser } from "./parsers/content_parser";
+export { CardResponseParser } from "./parsers/card_response_parser";
+
+// Configuration Types
+export type { CardGeneratorConfig } from "./configs/card_generator_config";
+export type { TypologyGeneratorConfig } from "./configs/typology_generator_config";
+export type { ConceptFactsGeneratorConfig } from "./configs/concept_facts_generator_config";
+export type { EmbeddingGeneratorConfig } from "./configs/embedding_generator_config";
+export type {
+  LocalConsolidatorConfig,
+  GlobalConsolidatorConfig,
+} from "./configs/consolidator_config";
+export type {
+  ContentParserConfig,
+  CardResponseParserConfig,
+} from "./configs/parser_config";
+
+// Default Configurations
+export { DEFAULT_CARD_GENERATOR_CONFIG } from "./configs/card_generator_config";
+export { DEFAULT_TYPOLOGY_GENERATOR_CONFIG } from "./configs/typology_generator_config";
+export { DEFAULT_CONCEPT_FACTS_GENERATOR_CONFIG } from "./configs/concept_facts_generator_config";
+export { DEFAULT_EMBEDDING_GENERATOR_CONFIG } from "./configs/embedding_generator_config";
+export {
+  DEFAULT_LOCAL_CONSOLIDATOR_CONFIG,
+  DEFAULT_GLOBAL_CONSOLIDATOR_CONFIG,
+} from "./configs/consolidator_config";
+export {
+  DEFAULT_CONTENT_PARSER_CONFIG,
+  DEFAULT_CARD_RESPONSE_PARSER_CONFIG,
+} from "./configs/parser_config";
+
+// Types
+export type {
+  ConceptFact,
+  ConceptWithEmbedding,
+} from "./embeddings/embedding_generator";
+export type {
+  ConceptWithId,
+  ConsolidationResult,
+} from "./consolidation/local_consolidator";
+export type { GlobalConsolidationResult } from "./consolidation/global_consolidator";
+export type { ParsedContent } from "./parsers/content_parser";
+
+// Services (keep existing)
+export { OpenAiService } from "./services/open_ai_service";
+export { default as qdrantClient } from "./services/qdrant_service";
+
+// Utilities
+export { ValidationError } from "./utils/validation";

package/src/parsers/card_response_parser.ts

@@ -0,0 +1,63 @@
+/**
+ * Standalone CardResponseParser module
+ * Parses and validates generated cards
+ */
+
+import { ParseCardResponse } from "../parse/parse_card_response";
+import {
+  CardResponseParserConfig,
+  DEFAULT_CARD_RESPONSE_PARSER_CONFIG,
+} from "../configs/parser_config";
+import { GeneratedCardResponseType } from "../types/raw_card_response_types/generated_card_response_type";
+import { SourceTaxonomy } from "../types/source_taxonomy_type";
+
+export class CardResponseParser {
+  private config: CardResponseParserConfig;
+
+  constructor(config: CardResponseParserConfig) {
+    this.validateConfig(config);
+    this.config = { ...DEFAULT_CARD_RESPONSE_PARSER_CONFIG, ...config };
+  }
+
+  /**
+   * Parse and validate generated card response
+   */
+  async parse(generatedData: GeneratedCardResponseType): Promise<any> {
+    const parser = new ParseCardResponse();
+
+    // Create source taxonomy from config
+    const sourceTaxonomy: SourceTaxonomy = {
+      concepts_facts: this.config.sourceTaxonomy.concepts_facts.map(
+        (concept) => ({
+          text: concept.text,
+          type: concept.type,
+          reference: concept.reference,
+          id: concept.id,
+        })
+      ),
+      fields: [],
+      learn_value: {
+        value: 50,
+        reason: "Generated for card parsing",
+        bloom_levels: [],
+        bloom_levels_reason: "",
+      },
+    };
+
+    return await parser.parse(
+      generatedData,
+      sourceTaxonomy,
+      this.config.bloomLevel!
+    );
+  }
+
+  private validateConfig(config: CardResponseParserConfig): void {
+    if (!config.sourceTaxonomy) {
+      throw new Error("Source taxonomy is required");
+    }
+
+    if (!config.sourceTaxonomy.concepts_facts) {
+      throw new Error("Concepts facts are required in source taxonomy");
+    }
+  }
+}

package/src/parsers/content_parser.ts

@@ -0,0 +1,90 @@
+/**
+ * Standalone ContentParser module
+ * Parses and sanitizes source content
+ */
+
+import { ParseSourceContent } from "../parse/parse_source_content";
+import {
+  ContentParserConfig,
+  DEFAULT_CONTENT_PARSER_CONFIG,
+} from "../configs/parser_config";
+import { validateContentType } from "../utils/validation";
+
+export interface ParsedContent {
+  source_id: string;
+  type: string;
+  title: string;
+  content: any[];
+  headings: string[];
+  taxonomy?: any;
+}
+
+export class ContentParser {
+  private config: ContentParserConfig;
+
+  constructor(config: ContentParserConfig) {
+    this.validateConfig(config);
+    this.config = { ...DEFAULT_CONTENT_PARSER_CONFIG, ...config };
+  }
+
+  /**
+   * Parse raw content into structured format
+   */
+  parse(rawContent: any): ParsedContent {
+    // Create a custom ParseSourceContent instance with our config
+    const parser = new ParseSourceContent(rawContent);
+
+    // Override the default titles and block types if provided
+    if (this.config.titlesToRemove) {
+      parser.titles_to_remove = this.config.titlesToRemove;
+    }
+
+    if (this.config.blockTypesToRemove) {
+      parser.block_types_toremove = this.config.blockTypesToRemove;
+    }
+
+    const parsedData = parser.parseData();
+
+    return {
+      source_id: parsedData.source_id,
+      type: parsedData.type,
+      title: parsedData.title,
+      content: parsedData.content,
+      headings: parsedData.headings,
+      taxonomy: parsedData.taxonomy,
+    };
+  }
+
+  /**
+   * Parse video content with timecode collapsing
+   */
+  parseVideoContent(rawContent: any): ParsedContent {
+    const parser = new ParseSourceContent(rawContent);
+
+    // Override the collapse timecodes method if max duration is specified
+    if (this.config.maxTimecodeDuration) {
+      const originalCollapseTimeCodes = parser.collapseTimeCodes.bind(parser);
+      parser.collapseTimeCodes = (
+        data: any[],
+        maxDuration = this.config.maxTimecodeDuration
+      ) => {
+        return originalCollapseTimeCodes(data, maxDuration);
+      };
+    }
+
+    const parsedData = parser.parseData();
+
+    return {
+      source_id: parsedData.source_id,
+      type: parsedData.type,
+      title: parsedData.title,
+      content: parsedData.content,
+      headings: parsedData.headings,
+      taxonomy: parsedData.taxonomy,
+    };
+  }
+
+  private validateConfig(config: ContentParserConfig): void {
+    validateContentType(config.type);
+  }
+}