only_ever_generator 8.0.0 → 8.0.2

package/src/utils/validation.ts

@@ -0,0 +1,298 @@
+/**
+ * Input validation utilities for OnlyEver Generator
+ */
+
+export class ValidationError extends Error {
+  constructor(message: string, public field?: string) {
+    super(message);
+    this.name = "ValidationError";
+  }
+}
+
+/**
+ * Validates that a value is not null or undefined
+ */
+export function validateRequired<T>(
+  value: T | null | undefined,
+  fieldName: string
+): T {
+  if (value === null || value === undefined) {
+    throw new ValidationError(`${fieldName} is required`, fieldName);
+  }
+  return value;
+}
+
+/**
+ * Validates that a string is not empty
+ */
+export function validateNonEmptyString(
+  value: string | null | undefined,
+  fieldName: string
+): string {
+  const validated = validateRequired(value, fieldName);
+  if (typeof validated !== "string" || validated.trim().length === 0) {
+    throw new ValidationError(
+      `${fieldName} must be a non-empty string`,
+      fieldName
+    );
+  }
+  return validated;
+}
+
+/**
+ * Validates that a value is a positive number
+ */
+export function validatePositiveNumber(
+  value: number | null | undefined,
+  fieldName: string
+): number {
+  const validated = validateRequired(value, fieldName);
+  if (typeof validated !== "number" || validated <= 0) {
+    throw new ValidationError(
+      `${fieldName} must be a positive number`,
+      fieldName
+    );
+  }
+  return validated;
+}
+
+/**
+ * Validates that a number is within a range
+ */
+export function validateNumberRange(
+  value: number | null | undefined,
+  fieldName: string,
+  min: number,
+  max: number
+): number {
+  const validated = validateRequired(value, fieldName);
+  if (typeof validated !== "number" || validated < min || validated > max) {
+    throw new ValidationError(
+      `${fieldName} must be between ${min} and ${max}`,
+      fieldName
+    );
+  }
+  return validated;
+}
+
+/**
+ * Validates that a value is one of the allowed values
+ */
+export function validateEnum<T extends string>(
+  value: T | null | undefined,
+  fieldName: string,
+  allowedValues: readonly T[]
+): T {
+  const validated = validateRequired(value, fieldName);
+  if (!allowedValues.includes(validated)) {
+    throw new ValidationError(
+      `${fieldName} must be one of: ${allowedValues.join(", ")}`,
+      fieldName
+    );
+  }
+  return validated;
+}
+
+/**
+ * Validates that a value is an array with at least one element
+ */
+export function validateNonEmptyArray<T>(
+  value: T[] | null | undefined,
+  fieldName: string
+): T[] {
+  const validated = validateRequired(value, fieldName);
+  if (!Array.isArray(validated) || validated.length === 0) {
+    throw new ValidationError(
+      `${fieldName} must be a non-empty array`,
+      fieldName
+    );
+  }
+  return validated;
+}
+
+/**
+ * Validates OpenAI API key format
+ */
+export function validateOpenAIKey(apiKey: string | null | undefined): string {
+  const validated = validateNonEmptyString(apiKey, "OpenAI API key");
+
+  // Basic format validation for OpenAI API keys
+  if (!validated.startsWith("sk-")) {
+    throw new ValidationError('OpenAI API key must start with "sk-"', "apiKey");
+  }
+
+  if (validated.length < 20) {
+    throw new ValidationError(
+      "OpenAI API key appears to be too short",
+      "apiKey"
+    );
+  }
+
+  return validated;
+}
+
+/**
+ * Validates OpenAI model name
+ */
+export function validateOpenAIModel(model: string | null | undefined): string {
+  const validated = validateNonEmptyString(model, "OpenAI model");
+
+  // Common OpenAI model patterns
+  const validModels = [
+    "gpt-4o",
+    "gpt-4o-mini",
+    "gpt-4",
+    "gpt-4-turbo",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-16k",
+  ];
+
+  if (!validModels.some((validModel) => validated.startsWith(validModel))) {
+    console.warn(
+      `Model "${validated}" may not be a valid OpenAI model. Valid models include: ${validModels.join(
+        ", "
+      )}`
+    );
+  }
+
+  return validated;
+}
+
+/**
+ * Validates prompt ID format
+ */
+export function validatePromptId(
+  promptId: string | null | undefined,
+  fieldName: string
+): string {
+  const validated = validateNonEmptyString(promptId, fieldName);
+
+  // Basic format validation for prompt IDs
+  if (!validated.startsWith("pmpt_")) {
+    throw new ValidationError(
+      `${fieldName} must start with "pmpt_"`,
+      fieldName
+    );
+  }
+
+  if (validated.length < 10) {
+    throw new ValidationError(
+      `${fieldName} appears to be too short`,
+      fieldName
+    );
+  }
+
+  return validated;
+}
+
+/**
+ * Validates content type
+ */
+export function validateContentType(type: string | null | undefined): string {
+  return validateEnum(type, "content type", ["text", "video"] as const);
+}
+
+/**
+ * Validates source ID format
+ */
+export function validateSourceId(sourceId: string | null | undefined): string {
+  const validated = validateNonEmptyString(sourceId, "source ID");
+
+  // Basic validation - should be a reasonable length
+  if (validated.length < 3) {
+    throw new ValidationError(
+      "Source ID must be at least 3 characters long",
+      "sourceId"
+    );
+  }
+
+  return validated;
+}
+
+/**
+ * Validates similarity threshold
+ */
+export function validateSimilarityThreshold(
+  threshold: number | null | undefined,
+  fieldName: string
+): number {
+  return validateNumberRange(threshold, fieldName, 0, 1);
+}
+
+/**
+ * Validates embedding dimensions
+ */
+export function validateEmbeddingDimensions(
+  dimensions: number | null | undefined
+): number {
+  const validated = validateRequired(dimensions, "embedding dimensions");
+
+  // Common embedding dimensions
+  const validDimensions = [128, 256, 512, 768, 1024, 1536, 2048];
+
+  if (!validDimensions.includes(validated)) {
+    console.warn(
+      `Embedding dimensions ${validated} may not be standard. Common values: ${validDimensions.join(
+        ", "
+      )}`
+    );
+  }
+
+  return validated;
+}
+
+/**
+ * Validates that an object has required properties
+ */
+export function validateObjectProperties<T extends Record<string, any>>(
+  obj: T | null | undefined,
+  fieldName: string,
+  requiredProperties: (keyof T)[]
+): T {
+  const validated = validateRequired(obj, fieldName);
+
+  for (const prop of requiredProperties) {
+    if (
+      !(prop in validated) ||
+      validated[prop] === null ||
+      validated[prop] === undefined
+    ) {
+      throw new ValidationError(
+        `${fieldName} must have property "${String(prop)}"`,
+        fieldName
+      );
+    }
+  }
+
+  return validated;
+}
+
+/**
+ * Validates array of strings
+ */
+export function validateStringArray(
+  value: string[] | null | undefined,
+  fieldName: string,
+  allowEmpty: boolean = true
+): string[] {
+  const validated = validateRequired(value, fieldName);
+
+  if (!Array.isArray(validated)) {
+    throw new ValidationError(`${fieldName} must be an array`, fieldName);
+  }
+
+  if (!allowEmpty && validated.length === 0) {
+    throw new ValidationError(`${fieldName} must not be empty`, fieldName);
+  }
+
+  for (let i = 0; i < validated.length; i++) {
+    if (typeof validated[i] !== "string") {
+      throw new ValidationError(
+        `${fieldName}[${i}] must be a string`,
+        fieldName
+      );
+    }
+  }
+
+  return validated;
+}

package/src/bootstrap/app.ts

@@ -1,312 +0,0 @@
-import { GenerateCards } from "../card_gen/generate_cards";
-import { ParseSourceContent } from "../parse/parse_source_content";
-import { OpenAiService } from "../services/open_ai_service";
-import { GenerateTypology } from "../typology_gen/generate_typology";
-import { GenerateArgs } from "../utils/generate_args";
-import { GenerateEmbeddings } from "../embedding_generation/generate_embeddings";
-import { LocalConsolidation } from "../embedding_generation/consolidation/local_consolidation";
-import { GlobalConsolidation } from "../embedding_generation/consolidation/global_consolidation";
-import { WriteConsolidatedData } from "../embedding_generation/consolidation/write_consolidated_data";
-import { BaseParamType } from "../types/base_param_type";
-import { SourceTaxonomy } from "../types/source_taxonomy_type";
-import { MongoConceptFactSource } from "../types/mongo_concept_fact_type";
-import { OpenAIHelper } from "../helper/openai_helper";
-import { GenerateConceptFacts } from "../typology_gen/generate_concept_facts";
-
-export class OnlyEverGenerator {
-  public api_key: string = "";
-  public openAiService: OpenAiService;
-
-  /// these fields will be populated inside the constructor
-  parsedContent?: {
-    title: string;
-    headings: string[];
-    content: any[];
-    taxonomy: SourceTaxonomy;
-    source_id: string;
-    type: string;
-  } = undefined;
-  promptForTypology: string = "";
-  promptForCardGen: string = "";
-  public generationCurriculum: boolean = false;
-  public promptIdForTypology: string;
-  public promptIdForConceptFacts: string;
-  public promptIdForCardsGeneration: string;
-  public n: number = 0;
-  generationVariables: {
-    bloom_instructions: string;
-    card_instructions: string;
-    card_examples: string;
-  } = {
-    bloom_instructions: "",
-    card_instructions: "",
-    card_examples: "",
-  };
-  expectedFields: Array<string> = [];
-  typologyResponse?: any = undefined;
-  cardgenResponse: any = undefined;
-  summarizeResponse = {};
-
-  constructor(
-    apiKey: string,
-    model: string,
-    generationContent: BaseParamType,
-    promptIdForTypology: string,
-    promptIdForConceptFacts: string,
-    promptIdForCardsGeneration: string,
-    generationVariables: {
-      bloom_instructions: string;
-      card_instructions: string;
-      card_examples: string;
-    },
-    generationCurriculum: boolean,
-    n: number | undefined
-  ) {
-    this.promptIdForTypology = promptIdForTypology;
-    this.promptIdForConceptFacts = promptIdForConceptFacts;
-    this.promptIdForCardsGeneration = promptIdForCardsGeneration;
-    this.api_key = apiKey;
-    this.generationVariables = generationVariables;
-    this.generationCurriculum = generationCurriculum;
-    this.n = n ?? 1;
-    this.openAiService = new OpenAiService(
-      apiKey,
-      model ?? "gpt-3.5-turbo-1106"
-    );
-
-    const parsedData = new ParseSourceContent(
-      generationContent.content
-    ).parseData();
-    (this.parsedContent = {
-      title: parsedData.title,
-      headings: parsedData.headings,
-      content: parsedData.content,
-      taxonomy: parsedData.taxonomy,
-      source_id: parsedData.source_id,
-      type: parsedData.type,
-    }),
-      (this.typologyResponse = generationContent.content.taxonomy);
-
-    this.expectedFields = generationContent.content.fields; //returnFields();
-    this.promptForTypology = generationContent.prompt.typology;
-    this.promptForCardGen = generationContent.prompt.card_generation;
-  }
-
-  async generate(
-    generate_typology: boolean = false,
-    generate_card: boolean = false
-  ): Promise<Array<any>> {
-    try {
-      let args = new GenerateArgs(generate_card, generate_typology, false);
-      const responseToReturn = [];
-      const whatNeedsToBeGenerated = args.getWhatNeedsToBeGenerated();
-      for (let elem of whatNeedsToBeGenerated)
-        if (elem == "generate_tyopology") {
-          console.log("Generating typology");
-          this.typologyResponse = await this.generateTypology();
-          console.log("Typology generated");
-          console.log(this.typologyResponse?.generate_cards?.value);
-          console.log(
-            JSON.stringify(this.typologyResponse?.generate_cards?.value > 30)
-          );
-
-          if (
-            this.typologyResponse?.status_code == 200 &&
-            (this.typologyResponse?.generate_cards?.value ?? 30) > 30
-          ) {
-            console.log("Concept facts generating");
-            const response = await this.generateConceptFacts(
-              this.typologyResponse
-            );
-            this.typologyResponse.concepts_facts =
-              response?.concepts_facts ?? [];
-            this.typologyResponse.metadata?.push(response?.metadata ?? {});
-            console.log("Concept facts generated");
-            console.log(this.typologyResponse);
-            const embeddings = await this.generateEmbeddings(
-              this.typologyResponse
-            );
-            console.log("Embedding generated");
-            const localConsolidation = await this.localConsolidation(
-              embeddings.concepts_facts,
-              this.parsedContent!.title
-            );
-            console.log("Local consolidation done");
-            this.typologyResponse!.concepts_facts =
-              localConsolidation.sourceTaxonomyOps;
-            const globalConsolidation = await this.globalConsolidation(
-              localConsolidation.sourceTaxonomyOps,
-              this.parsedContent!.source_id,
-              0.8
-            );
-            console.log("Global consolidation done");
-            this.typologyResponse!.concepts_facts =
-              globalConsolidation.source_taxonomy;
-            this.typologyResponse!.metadata?.push(embeddings.metadata);
-            responseToReturn.push({
-              type: "typology",
-              data: this.typologyResponse,
-              qdrantOps: {
-                update: globalConsolidation.global_updates,
-                inserts: globalConsolidation.global_inserts,
-              },
-            });
-          } else {
-            responseToReturn.push({
-              type: "typology",
-              data: this.typologyResponse,
-            });
-          }
-        } else if (elem == "generate_card") {
-          if (this.shouldTheCardBeGeneratedAfterTypologyResponse()) {
-            console.log("Generating card");
-            this.parsedContent!.taxonomy = {
-              concepts_facts: this.typologyResponse?.concepts_facts ?? [],
-              fields: this.expectedFields,
-              learn_value: this.typologyResponse?.learn_value ?? {
-                value: 0,
-                reason: "",
-                bloom_levels: [],
-                bloom_levels_reason: "",
-              },
-            };
-            this.cardgenResponse = await this.generateCard(this.n);
-            responseToReturn.push({
-              type: "card_gen",
-              data: this.cardgenResponse,
-            });
-          }
-        }
-      return responseToReturn;
-    } catch (error) {
-      console.log(error);
-      return [];
-    }
-  }
-
-  shouldTheCardBeGeneratedAfterTypologyResponse() {
-    if (this.typologyResponse) {
-      return (
-        (this.typologyResponse?.generate_cards?.value ??
-          this.typologyResponse?.learn_value?.value ??
-          30) > 30
-      );
-    } else {
-      return false;
-    }
-  }
-
-  async generateCard(n: number) {
-    const content = {
-      title: this.parsedContent?.title,
-      concepts_facts: this.typologyResponse?.concepts_facts,
-      content: this.parsedContent?.content,
-    };
-    let generateCardsResp = await new GenerateCards(
-      this.openAiService,
-      this.parsedContent?.source_id ?? "",
-      JSON.stringify(content),
-      this.parsedContent!.taxonomy,
-      this.generationVariables,
-      this.promptIdForCardsGeneration,
-      this.generationCurriculum
-    ).generateCards(n);
-    return generateCardsResp;
-  }
-
-  async generateTypology() {
-    const type = this.parsedContent?.type ?? "text";
-    const content = {
-      title: this.parsedContent?.title ?? "",
-      ...(type === "text"
-        ? { h1_headings: this.parsedContent?.headings ?? [""] }
-        : { timecodes: this.parsedContent?.headings ?? [""] }),
-      content: this.parsedContent?.content ?? [],
-    };
-    try {
-      let response = await new GenerateTypology(
-        this.openAiService,
-        type,
-        this.parsedContent?.source_id ?? "",
-        content,
-        this.expectedFields,
-        this.promptIdForTypology,
-        this.generationCurriculum
-      ).generate();
-      return response;
-    } catch (error) {
-      console.log(error);
-    }
-  }
-
-  async generateConceptFacts(typologyResponse: any) {
-    const type = this.parsedContent?.type ?? "text";
-    const content = {
-      title: this.parsedContent?.title ?? "",
-      ...(type === "text"
-        ? { h1_headings: this.parsedContent?.headings ?? [""] }
-        : { timecodes: this.parsedContent?.headings ?? [""] }),
-      key_takeaways: typologyResponse?.summary_cards ?? [],
-      content: this.parsedContent?.content ?? [],
-      ...(typologyResponse && typeof typologyResponse === "object"
-        ? {
-            fields: typologyResponse.field ?? [],
-          }
-        : {}),
-    };
-    let response = await new GenerateConceptFacts(
-      this.openAiService,
-      this.parsedContent?.source_id ?? "",
-      content,
-      type,
-      this.promptIdForConceptFacts,
-      this.generationCurriculum
-    ).generate();
-    return response;
-  }
-
-  async generateEmbeddings(typologyResponse: any) {
-    const concepts_facts = typologyResponse.concepts_facts;
-    const texts = concepts_facts.map((e: MongoConceptFactSource) => e.text);
-    const embeddings = await new GenerateEmbeddings(
-      this.openAiService
-    ).generateEmbeddings(concepts_facts);
-    return embeddings;
-  }
-
-  async localConsolidation(
-    concepts_facts: {
-      text: string;
-      type: string;
-      reference: string;
-      embedding: number[];
-    }[],
-    sourceId: string
-  ) {
-    const embeddings = new LocalConsolidation().consolidate(
-      concepts_facts,
-      sourceId
-    );
-    return embeddings;
-  }
-
-  async globalConsolidation(
-    concepts_facts: {
-      id: string;
-      text: string;
-      type: string;
-      reference: string;
-      embedding: number[];
-    }[],
-    sourceId: string,
-    threshold: number
-  ) {
-    const embeddings = await new GlobalConsolidation().consolidate(
-      concepts_facts,
-      sourceId,
-      threshold ?? 0.8
-    );
-
-    return embeddings;
-  }
-}