only_ever_generator 8.0.2 → 8.0.4

package/dist/configs/concept_facts_generator_config.js

@@ -1,12 +0,0 @@
-"use strict";
-/**
- * Configuration for ConceptFactsGenerator module
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DEFAULT_CONCEPT_FACTS_GENERATOR_CONFIG = void 0;
-exports.DEFAULT_CONCEPT_FACTS_GENERATOR_CONFIG = {
-    model: "gpt-4o-mini",
-    generationCurriculum: false,
-    type: "text",
-};
-//# sourceMappingURL=concept_facts_generator_config.js.map

package/dist/configs/concept_facts_generator_config.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"concept_facts_generator_config.js","sourceRoot":"","sources":["../../src/configs/concept_facts_generator_config.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAgCU,QAAA,sCAAsC,GACjD;IACE,KAAK,EAAE,aAAa;IACpB,oBAAoB,EAAE,KAAK;IAC3B,IAAI,EAAE,MAAM;CACb,CAAC"}

package/dist/configs/consolidator_config.d.ts

@@ -1,20 +0,0 @@
-/**
- * Configuration for Consolidator modules
- */
-export interface LocalConsolidatorConfig {
-    /** Similarity threshold for duplicate detection (0-1) */
-    threshold?: number;
-}
-export interface GlobalConsolidatorConfig {
-    /** Qdrant vector database URL */
-    qdrantUrl: string;
-    /** Qdrant API key */
-    qdrantApiKey: string;
-    /** Similarity threshold for global matching (0-1) */
-    threshold?: number;
-    /** Collection name in Qdrant */
-    collectionName?: string;
-}
-export declare const DEFAULT_LOCAL_CONSOLIDATOR_CONFIG: Partial<LocalConsolidatorConfig>;
-export declare const DEFAULT_GLOBAL_CONSOLIDATOR_CONFIG: Partial<GlobalConsolidatorConfig>;
-//# sourceMappingURL=consolidator_config.d.ts.map

package/dist/configs/consolidator_config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"consolidator_config.d.ts","sourceRoot":"","sources":["../../src/configs/consolidator_config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,uBAAuB;IACtC,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,wBAAwB;IACvC,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;IAElB,qBAAqB;IACrB,YAAY,EAAE,MAAM,CAAC;IAErB,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gCAAgC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,eAAO,MAAM,iCAAiC,EAAE,OAAO,CAAC,uBAAuB,CAG5E,CAAC;AAEJ,eAAO,MAAM,kCAAkC,EAAE,OAAO,CAAC,wBAAwB,CAI9E,CAAC"}

package/dist/configs/consolidator_config.js

@@ -1,14 +0,0 @@
-"use strict";
-/**
- * Configuration for Consolidator modules
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DEFAULT_GLOBAL_CONSOLIDATOR_CONFIG = exports.DEFAULT_LOCAL_CONSOLIDATOR_CONFIG = void 0;
-exports.DEFAULT_LOCAL_CONSOLIDATOR_CONFIG = {
-    threshold: 0.65,
-};
-exports.DEFAULT_GLOBAL_CONSOLIDATOR_CONFIG = {
-    threshold: 0.8,
-    collectionName: "concepts-vectors",
-};
-//# sourceMappingURL=consolidator_config.js.map

package/dist/configs/consolidator_config.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"consolidator_config.js","sourceRoot":"","sources":["../../src/configs/consolidator_config.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAqBU,QAAA,iCAAiC,GAC5C;IACE,SAAS,EAAE,IAAI;CAChB,CAAC;AAES,QAAA,kCAAkC,GAC7C;IACE,SAAS,EAAE,GAAG;IACd,cAAc,EAAE,kBAAkB;CACnC,CAAC"}

package/dist/configs/embedding_generator_config.d.ts

@@ -1,13 +0,0 @@
-/**
- * Configuration for EmbeddingGenerator module
- */
-export interface EmbeddingGeneratorConfig {
-    /** OpenAI API key */
-    apiKey: string;
-    /** Embedding model to use (e.g., "text-embedding-3-large") */
-    model?: string;
-    /** Embedding dimensions */
-    dimensions?: number;
-}
-export declare const DEFAULT_EMBEDDING_GENERATOR_CONFIG: Partial<EmbeddingGeneratorConfig>;
-//# sourceMappingURL=embedding_generator_config.d.ts.map

package/dist/configs/embedding_generator_config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"embedding_generator_config.d.ts","sourceRoot":"","sources":["../../src/configs/embedding_generator_config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,wBAAwB;IACvC,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IAEf,8DAA8D;IAC9D,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,kCAAkC,EAAE,OAAO,CAAC,wBAAwB,CAI9E,CAAC"}

package/dist/configs/embedding_generator_config.js

@@ -1,11 +0,0 @@
-"use strict";
-/**
- * Configuration for EmbeddingGenerator module
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DEFAULT_EMBEDDING_GENERATOR_CONFIG = void 0;
-exports.DEFAULT_EMBEDDING_GENERATOR_CONFIG = {
-    model: "text-embedding-3-large",
-    dimensions: 512,
-};
-//# sourceMappingURL=embedding_generator_config.js.map

package/dist/configs/embedding_generator_config.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"embedding_generator_config.js","sourceRoot":"","sources":["../../src/configs/embedding_generator_config.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAaU,QAAA,kCAAkC,GAC7C;IACE,KAAK,EAAE,wBAAwB;IAC/B,UAAU,EAAE,GAAG;CAChB,CAAC"}

package/dist/configs/parser_config.d.ts

@@ -1,31 +0,0 @@
-/**
- * Configuration for Parser modules
- */
-export interface ContentParserConfig {
-    /** Content type to parse */
-    type: "text" | "video";
-    /** Whether to remove unwanted sections */
-    removeUnwantedSections?: boolean;
-    /** Custom titles to remove */
-    titlesToRemove?: string[];
-    /** Block types to remove */
-    blockTypesToRemove?: string[];
-    /** Maximum duration for video timecode collapsing (seconds) */
-    maxTimecodeDuration?: number;
-}
-export interface CardResponseParserConfig {
-    /** Source taxonomy for concept/fact validation */
-    sourceTaxonomy: {
-        concepts_facts: Array<{
-            id: string;
-            text: string;
-            type: "concept" | "fact";
-            reference: string;
-        }>;
-    };
-    /** Bloom level for cards */
-    bloomLevel?: number;
-}
-export declare const DEFAULT_CONTENT_PARSER_CONFIG: Partial<ContentParserConfig>;
-export declare const DEFAULT_CARD_RESPONSE_PARSER_CONFIG: Partial<CardResponseParserConfig>;
-//# sourceMappingURL=parser_config.d.ts.map

package/dist/configs/parser_config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parser_config.d.ts","sourceRoot":"","sources":["../../src/configs/parser_config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,mBAAmB;IAClC,4BAA4B;IAC5B,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAEvB,0CAA0C;IAC1C,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAEjC,8BAA8B;IAC9B,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAE1B,4BAA4B;IAC5B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,+DAA+D;IAC/D,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,wBAAwB;IACvC,kDAAkD;IAClD,cAAc,EAAE;QACd,cAAc,EAAE,KAAK,CAAC;YACpB,EAAE,EAAE,MAAM,CAAC;YACX,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;YACzB,SAAS,EAAE,MAAM,CAAC;SACnB,CAAC,CAAC;KACJ,CAAC;IAEF,4BAA4B;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,6BAA6B,EAAE,OAAO,CAAC,mBAAmB,CAetE,CAAC;AAEF,eAAO,MAAM,mCAAmC,EAAE,OAAO,CAAC,wBAAwB,CAG/E,CAAC"}

package/dist/configs/parser_config.js

@@ -1,26 +0,0 @@
-"use strict";
-/**
- * Configuration for Parser modules
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DEFAULT_CARD_RESPONSE_PARSER_CONFIG = exports.DEFAULT_CONTENT_PARSER_CONFIG = void 0;
-exports.DEFAULT_CONTENT_PARSER_CONFIG = {
-    type: "text",
-    removeUnwantedSections: true,
-    titlesToRemove: [
-        "See also",
-        "References",
-        "Further reading",
-        "External links",
-        "Notes and references",
-        "Bibliography",
-        "Notes",
-        "Cited sources",
-    ],
-    blockTypesToRemove: ["table", "empty_line"],
-    maxTimecodeDuration: 30.0,
-};
-exports.DEFAULT_CARD_RESPONSE_PARSER_CONFIG = {
-    bloomLevel: 1,
-};
-//# sourceMappingURL=parser_config.js.map

package/dist/configs/parser_config.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parser_config.js","sourceRoot":"","sources":["../../src/configs/parser_config.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AAkCU,QAAA,6BAA6B,GAAiC;IACzE,IAAI,EAAE,MAAM;IACZ,sBAAsB,EAAE,IAAI;IAC5B,cAAc,EAAE;QACd,UAAU;QACV,YAAY;QACZ,iBAAiB;QACjB,gBAAgB;QAChB,sBAAsB;QACtB,cAAc;QACd,OAAO;QACP,eAAe;KAChB;IACD,kBAAkB,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC;IAC3C,mBAAmB,EAAE,IAAI;CAC1B,CAAC;AAEW,QAAA,mCAAmC,GAC9C;IACE,UAAU,EAAE,CAAC;CACd,CAAC"}

package/dist/configs/typology_generator_config.d.ts

@@ -1,26 +0,0 @@
-/**
- * Configuration for TypologyGenerator module
- */
-export interface TypologyGeneratorConfig {
-    /** OpenAI API key */
-    apiKey: string;
-    /** OpenAI model to use (e.g., "gpt-4o-mini", "gpt-4") */
-    model: string;
-    /** Content to analyze */
-    content: {
-        title: string;
-        h1_headings?: string[];
-        timecodes?: string[];
-        content: any[];
-    };
-    /** Expected fields for classification */
-    expectedFields: string[];
-    /** OpenAI prompt ID for typology generation */
-    promptId: string;
-    /** Whether this is curriculum-based generation */
-    generationCurriculum?: boolean;
-    /** Source ID for tracking */
-    sourceId?: string;
-}
-export declare const DEFAULT_TYPOLOGY_GENERATOR_CONFIG: Partial<TypologyGeneratorConfig>;
-//# sourceMappingURL=typology_generator_config.d.ts.map

package/dist/configs/typology_generator_config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"typology_generator_config.d.ts","sourceRoot":"","sources":["../../src/configs/typology_generator_config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,uBAAuB;IACtC,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IAEf,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IAEd,yBAAyB;IACzB,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,OAAO,EAAE,GAAG,EAAE,CAAC;KAChB,CAAC;IAEF,yCAAyC;IACzC,cAAc,EAAE,MAAM,EAAE,CAAC;IAEzB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IAEjB,kDAAkD;IAClD,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B,6BAA6B;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,iCAAiC,EAAE,OAAO,CAAC,uBAAuB,CAK5E,CAAC"}

package/dist/configs/typology_generator_config.js

@@ -1,12 +0,0 @@
-"use strict";
-/**
- * Configuration for TypologyGenerator module
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DEFAULT_TYPOLOGY_GENERATOR_CONFIG = void 0;
-exports.DEFAULT_TYPOLOGY_GENERATOR_CONFIG = {
-    model: "gpt-4o-mini",
-    generationCurriculum: false,
-    expectedFields: [],
-};
-//# sourceMappingURL=typology_generator_config.js.map

package/dist/configs/typology_generator_config.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"typology_generator_config.js","sourceRoot":"","sources":["../../src/configs/typology_generator_config.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AA8BU,QAAA,iCAAiC,GAC5C;IACE,KAAK,EAAE,aAAa;IACpB,oBAAoB,EAAE,KAAK;IAC3B,cAAc,EAAE,EAAE;CACnB,CAAC"}

package/dist/consolidation/global_consolidator.d.ts

@@ -1,47 +0,0 @@
-/**
- * Standalone GlobalConsolidator module
- * Matches concepts against global Qdrant vector database
- */
-import { GlobalConsolidatorConfig } from "../configs/consolidator_config";
-export interface ConceptWithId {
-    text: string;
-    type: string;
-    reference: string;
-    id: string;
-    embedding: number[];
-}
-export interface GlobalConsolidationResult {
-    source_taxonomy: Array<{
-        text: string;
-        type: string;
-        reference: string;
-        id: string;
-    }>;
-    global_updates: Array<{
-        id: string;
-        sourceIdToAdd: string;
-        vector: number[];
-        currentPayload: {
-            _sources: string[];
-            text: string;
-        };
-    }>;
-    global_inserts: Array<{
-        id: string;
-        vector: number[];
-        payload: {
-            _sources: string[];
-            text: string;
-        };
-    }>;
-}
-export declare class GlobalConsolidator {
-    private config;
-    constructor(config: GlobalConsolidatorConfig);
-    /**
-     * Consolidate concepts against global database
-     */
-    consolidate(locally_consolidated_concepts_facts: ConceptWithId[], sourceId: string): Promise<GlobalConsolidationResult>;
-    private validateConfig;
-}
-//# sourceMappingURL=global_consolidator.d.ts.map

package/dist/consolidation/global_consolidator.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"global_consolidator.d.ts","sourceRoot":"","sources":["../../src/consolidation/global_consolidator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EACL,wBAAwB,EAEzB,MAAM,gCAAgC,CAAC;AAMxC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,yBAAyB;IACxC,eAAe,EAAE,KAAK,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,EAAE,EAAE,MAAM,CAAC;KACZ,CAAC,CAAC;IACH,cAAc,EAAE,KAAK,CAAC;QACpB,EAAE,EAAE,MAAM,CAAC;QACX,aAAa,EAAE,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,cAAc,EAAE;YACd,QAAQ,EAAE,MAAM,EAAE,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC;SACd,CAAC;KACH,CAAC,CAAC;IACH,cAAc,EAAE,KAAK,CAAC;QACpB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,OAAO,EAAE;YACP,QAAQ,EAAE,MAAM,EAAE,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC;SACd,CAAC;KACH,CAAC,CAAC;CACJ;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAA2B;gBAE7B,MAAM,EAAE,wBAAwB;IAK5C;;OAEG;IACG,WAAW,CACf,mCAAmC,EAAE,aAAa,EAAE,EACpD,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,yBAAyB,CAAC;IAkFrC,OAAO,CAAC,cAAc;CAYvB"}

package/dist/consolidation/global_consolidator.js

@@ -1,94 +0,0 @@
-"use strict";
-/**
- * Standalone GlobalConsolidator module
- * Matches concepts against global Qdrant vector database
- */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.GlobalConsolidator = void 0;
-const qdrant_db_methods_1 = require("../helper/qdrant_db_methods");
-const consolidator_config_1 = require("../configs/consolidator_config");
-const validation_1 = require("../utils/validation");
-class GlobalConsolidator {
-    constructor(config) {
-        this.validateConfig(config);
-        this.config = Object.assign(Object.assign({}, consolidator_config_1.DEFAULT_GLOBAL_CONSOLIDATOR_CONFIG), config);
-    }
-    /**
-     * Consolidate concepts against global database
-     */
-    consolidate(locally_consolidated_concepts_facts, sourceId) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
-            // For all concepts_facts, find the most similar concepts_facts in qdrant, threshold of 0.8
-            const similarConcepts = yield (0, qdrant_db_methods_1.getSimilarConcepts)(this.config.collectionName, locally_consolidated_concepts_facts, this.config.threshold);
-            const taxonomyConcepts = [];
-            const globalUpdatesOps = [];
-            const globalInsertsOps = [];
-            for (const index in similarConcepts) {
-                const points = similarConcepts[index].points;
-                const originalConcept = locally_consolidated_concepts_facts[index];
-                if (points.length == 0) {
-                    taxonomyConcepts.push({
-                        id: originalConcept.id,
-                        text: originalConcept.text,
-                        type: originalConcept.type,
-                        reference: originalConcept.reference,
-                    });
-                    globalInsertsOps.push({
-                        id: originalConcept.id,
-                        vector: originalConcept.embedding,
-                        payload: {
-                            _sources: [sourceId],
-                            text: originalConcept.text,
-                        },
-                    });
-                }
-                else {
-                    const consolidatedId = points[0].id;
-                    const currentPayload = points[0].payload;
-                    taxonomyConcepts.push({
-                        id: consolidatedId.toString(),
-                        text: originalConcept.text,
-                        type: originalConcept.type,
-                        reference: originalConcept.reference,
-                    });
-                    globalUpdatesOps.push({
-                        id: consolidatedId.toString(),
-                        sourceIdToAdd: sourceId,
-                        vector: originalConcept.embedding,
-                        currentPayload: {
-                            _sources: ((_a = currentPayload === null || currentPayload === void 0 ? void 0 : currentPayload._sources) !== null && _a !== void 0 ? _a : []),
-                            text: ((_b = currentPayload === null || currentPayload === void 0 ? void 0 : currentPayload.text) !== null && _b !== void 0 ? _b : "").toString(),
-                        },
-                    });
-                }
-            }
-            return {
-                source_taxonomy: taxonomyConcepts,
-                global_updates: globalUpdatesOps,
-                global_inserts: globalInsertsOps,
-            };
-        });
-    }
-    validateConfig(config) {
-        (0, validation_1.validateNonEmptyString)(config.qdrantUrl, "Qdrant URL");
-        (0, validation_1.validateNonEmptyString)(config.qdrantApiKey, "Qdrant API key");
-        if (config.threshold !== undefined) {
-            (0, validation_1.validateSimilarityThreshold)(config.threshold, "threshold");
-        }
-        if (config.collectionName !== undefined) {
-            (0, validation_1.validateNonEmptyString)(config.collectionName, "collection name");
-        }
-    }
-}
-exports.GlobalConsolidator = GlobalConsolidator;
-//# sourceMappingURL=global_consolidator.js.map

package/dist/consolidation/global_consolidator.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"global_consolidator.js","sourceRoot":"","sources":["../../src/consolidation/global_consolidator.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;AAEH,mEAAiE;AACjE,wEAGwC;AACxC,oDAG6B;AAoC7B,MAAa,kBAAkB;IAG7B,YAAY,MAAgC;QAC1C,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,mCAAQ,wDAAkC,GAAK,MAAM,CAAE,CAAC;IACrE,CAAC;IAED;;OAEG;IACG,WAAW,CACf,mCAAoD,EACpD,QAAgB;;;YAEhB,2FAA2F;YAC3F,MAAM,eAAe,GAAG,MAAM,IAAA,sCAAkB,EAC9C,IAAI,CAAC,MAAM,CAAC,cAAe,EAC3B,mCAAmC,EACnC,IAAI,CAAC,MAAM,CAAC,SAAU,CACvB,CAAC;YAEF,MAAM,gBAAgB,GAKjB,EAAE,CAAC;YAER,MAAM,gBAAgB,GAQjB,EAAE,CAAC;YAER,MAAM,gBAAgB,GAOjB,EAAE,CAAC;YAER,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;gBACpC,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;gBAC7C,MAAM,eAAe,GAAG,mCAAmC,CAAC,KAAK,CAAC,CAAC;gBAEnE,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBACvB,gBAAgB,CAAC,IAAI,CAAC;wBACpB,EAAE,EAAE,eAAe,CAAC,EAAE;wBACtB,IAAI,EAAE,eAAe,CAAC,IAAI;wBAC1B,IAAI,EAAE,eAAe,CAAC,IAAI;wBAC1B,SAAS,EAAE,eAAe,CAAC,SAAS;qBACrC,CAAC,CAAC;oBACH,gBAAgB,CAAC,IAAI,CAAC;wBACpB,EAAE,EAAE,eAAe,CAAC,EAAE;wBACtB,MAAM,EAAE,eAAe,CAAC,SAAS;wBACjC,OAAO,EAAE;4BACP,QAAQ,EAAE,CAAC,QAAQ,CAAC;4BACpB,IAAI,EAAE,eAAe,CAAC,IAAI;yBAC3B;qBACF,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;oBAEzC,gBAAgB,CAAC,IAAI,CAAC;wBACpB,EAAE,EAAE,cAAc,CAAC,QAAQ,EAAE;wBAC7B,IAAI,EAAE,eAAe,CAAC,IAAI;wBAC1B,IAAI,EAAE,eAAe,CAAC,IAAI;wBAC1B,SAAS,EAAE,eAAe,CAAC,SAAS;qBACrC,CAAC,CAAC;oBACH,gBAAgB,CAAC,IAAI,CAAC;wBACpB,EAAE,EAAE,cAAc,CAAC,QAAQ,EAAE;wBAC7B,aAAa,EAAE,QAAQ;wBACvB,MAAM,EAAE,eAAe,CAAC,SAAS;wBACjC,cAAc,EAAE;4BACd,QAAQ,EAAE,CAAC,MAAA,cAAc,aAAd,cAAc,uBAAd,cAAc,CAAE,QAAQ,mCAAI,EAAE,CAAa;4BACtD,IAAI,EAAE,CAAC,MAAA,cAAc,aAAd,cAAc,uBAAd,cAAc,CAAE,IAAI,mCAAI,EAAE,CAAC,CAAC,QAAQ,EAAE;yBAC9C;qBACF,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,eAAe,EAAE,gBAAgB;gBACjC,cAAc,EAAE,gBAAgB;gBAChC,cAAc,EAAE,gBAAgB;aACjC,CAAC;QACJ,CAAC;KAAA;IAEO,cAAc,CAAC,MAAgC;QACrD,IAAA,mCAAsB,EAAC,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACvD,IAAA,mCAAsB,EAAC,MAAM,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;QAE9D,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACnC,IAAA,wCAA2B,EAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACxC,IAAA,mCAAsB,EAAC,MAAM,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;CACF;AA5GD,gDA4GC"}

package/dist/consolidation/local_consolidator.d.ts

@@ -1,52 +0,0 @@
-/**
- * Standalone LocalConsolidator module
- * Removes duplicate concepts within a single source using vector similarity
- */
-import { LocalConsolidatorConfig } from "../configs/consolidator_config";
-export interface ConceptWithEmbedding {
-    text: string;
-    type: string;
-    embedding: number[];
-    reference: string;
-}
-export interface ConceptWithId {
-    id: string;
-    text: string;
-    type: string;
-    embedding: number[];
-    reference: string;
-}
-export interface ConsolidationResult {
-    globalConceptOps: Array<{
-        id: string;
-        vector: number[];
-        payload: {
-            _sources: string[];
-            text: string;
-        };
-    }>;
-    sourceIds: string[];
-    consolidatedConcepts: any[];
-    sourceTaxonomyOps: ConceptWithId[];
-}
-export declare class LocalConsolidator {
-    private config;
-    constructor(config?: LocalConsolidatorConfig);
-    /**
-     * Consolidate similar concepts within a single source
-     */
-    consolidate(concepts_facts: ConceptWithEmbedding[], sourceId: string): ConsolidationResult;
-    /**
-     * Calculate cosine similarity between two vectors
-     */
-    cosineSimilarity: (a: number[], b: number[]) => number;
-    /**
-     * Consolidate similar embeddings using cosine similarity
-     */
-    consolidateSimilarEmbeddings(data: ConceptWithId[], threshold?: number): {
-        finalConsolidatedConcepts: ConceptWithId[];
-        consolidatedConcepts: any[];
-    };
-    private validateConfig;
-}
-//# sourceMappingURL=local_consolidator.d.ts.map

package/dist/consolidation/local_consolidator.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"local_consolidator.d.ts","sourceRoot":"","sources":["../../src/consolidation/local_consolidator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EACL,uBAAuB,EAExB,MAAM,gCAAgC,CAAC;AAGxC,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,gBAAgB,EAAE,KAAK,CAAC;QACtB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,OAAO,EAAE;YACP,QAAQ,EAAE,MAAM,EAAE,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC;SACd,CAAC;KACH,CAAC,CAAC;IACH,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,oBAAoB,EAAE,GAAG,EAAE,CAAC;IAC5B,iBAAiB,EAAE,aAAa,EAAE,CAAC;CACpC;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAA0B;gBAE5B,MAAM,GAAE,uBAA4B;IAKhD;;OAEG;IACH,WAAW,CACT,cAAc,EAAE,oBAAoB,EAAE,EACtC,QAAQ,EAAE,MAAM,GACf,mBAAmB;IA8DtB;;OAEG;IACH,gBAAgB,GAAI,GAAG,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,KAAG,MAAM,CAKnD;IAEF;;OAEG;IACH,4BAA4B,CAAC,IAAI,EAAE,aAAa,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM;;;;IAqCtE,OAAO,CAAC,cAAc;CAKvB"}

package/dist/consolidation/local_consolidator.js

@@ -1,118 +0,0 @@
-"use strict";
-/**
- * Standalone LocalConsolidator module
- * Removes duplicate concepts within a single source using vector similarity
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalConsolidator = void 0;
-const crypto_1 = require("crypto");
-const consolidator_config_1 = require("../configs/consolidator_config");
-const validation_1 = require("../utils/validation");
-class LocalConsolidator {
-    constructor(config = {}) {
-        /**
-         * Calculate cosine similarity between two vectors
-         */
-        this.cosineSimilarity = (a, b) => {
-            const dotProduct = a.reduce((acc, val, index) => acc + val * b[index], 0);
-            const magnitudeA = Math.sqrt(a.reduce((acc, val) => acc + val * val, 0));
-            const magnitudeB = Math.sqrt(b.reduce((acc, val) => acc + val * val, 0));
-            return dotProduct / (magnitudeA * magnitudeB);
-        };
-        this.validateConfig(config);
-        this.config = Object.assign(Object.assign({}, consolidator_config_1.DEFAULT_LOCAL_CONSOLIDATOR_CONFIG), config);
-    }
-    /**
-     * Consolidate similar concepts within a single source
-     */
-    consolidate(concepts_facts, sourceId) {
-        try {
-            const sourceIds = [];
-            const sourceTaxonomyOps = [];
-            const globalConceptOps = [];
-            const consolidatedConcepts = [];
-            const concepts = concepts_facts.map((e) => {
-                return {
-                    id: (0, crypto_1.randomUUID)().toString(),
-                    text: e.text,
-                    reference: e.reference,
-                    embedding: e.embedding,
-                    type: e.type,
-                };
-            });
-            const afterConsolidation = this.consolidateSimilarEmbeddings(concepts);
-            sourceIds.push(JSON.stringify(sourceId));
-            console.log(afterConsolidation);
-            for (var concept of afterConsolidation.finalConsolidatedConcepts) {
-                sourceTaxonomyOps.push({
-                    text: concept.text,
-                    id: concept.id,
-                    embedding: concept.embedding,
-                    type: concept.type,
-                    reference: concept.reference,
-                });
-                globalConceptOps.push({
-                    id: concept.id.toString(),
-                    vector: concept.embedding,
-                    payload: {
-                        _sources: [sourceId],
-                        text: concept.text,
-                    },
-                });
-            }
-            consolidatedConcepts.push(...afterConsolidation.consolidatedConcepts);
-            return {
-                globalConceptOps: globalConceptOps,
-                sourceIds: sourceIds,
-                consolidatedConcepts: consolidatedConcepts,
-                sourceTaxonomyOps: sourceTaxonomyOps,
-            };
-        }
-        catch (e) {
-            console.error("Error occurred while converting the parsed array to db operations:", e);
-            throw e;
-        }
-    }
-    /**
-     * Consolidate similar embeddings using cosine similarity
-     */
-    consolidateSimilarEmbeddings(data, threshold) {
-        const finalConsolidatedConcepts = [];
-        const visited = new Array(data.length).fill(false);
-        const consolidatedConcepts = [];
-        const similarityThreshold = threshold !== null && threshold !== void 0 ? threshold : this.config.threshold;
-        for (let i = 0; i < data.length; i++) {
-            if (visited[i])
-                continue;
-            const group = [data[i]];
-            visited[i] = true;
-            for (let j = i + 1; j < data.length; j++) {
-                if (visited[j])
-                    continue;
-                const sim = this.cosineSimilarity(data[i].embedding, data[j].embedding);
-                if (sim > similarityThreshold) {
-                    group.push(data[j]);
-                    visited[j] = true;
-                    consolidatedConcepts.push({
-                        concept1: data[i].text,
-                        concept2: data[j].text,
-                        similarity: sim,
-                    });
-                }
-            }
-            // Consolidate the group (e.g., just take the first, or merge)
-            finalConsolidatedConcepts.push(group[0]); // Or you can customize how to merge
-        }
-        return {
-            finalConsolidatedConcepts: finalConsolidatedConcepts,
-            consolidatedConcepts: consolidatedConcepts,
-        };
-    }
-    validateConfig(config) {
-        if (config.threshold !== undefined) {
-            (0, validation_1.validateSimilarityThreshold)(config.threshold, "threshold");
-        }
-    }
-}
-exports.LocalConsolidator = LocalConsolidator;
-//# sourceMappingURL=local_consolidator.js.map

package/dist/consolidation/local_consolidator.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"local_consolidator.js","sourceRoot":"","sources":["../../src/consolidation/local_consolidator.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,mCAA0C;AAC1C,wEAGwC;AACxC,oDAAkE;AA+BlE,MAAa,iBAAiB;IAG5B,YAAY,SAAkC,EAAE;QAyEhD;;WAEG;QACH,qBAAgB,GAAG,CAAC,CAAW,EAAE,CAAW,EAAU,EAAE;YACtD,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;YAC1E,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACzE,OAAO,UAAU,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC;QAChD,CAAC,CAAC;QAhFA,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,mCAAQ,uDAAiC,GAAK,MAAM,CAAE,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,WAAW,CACT,cAAsC,EACtC,QAAgB;QAEhB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,EAAE,CAAC;YACrB,MAAM,iBAAiB,GAAoB,EAAE,CAAC;YAC9C,MAAM,gBAAgB,GAOjB,EAAE,CAAC;YACR,MAAM,oBAAoB,GAAU,EAAE,CAAC;YAEvC,MAAM,QAAQ,GAAoB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE;gBAC9D,OAAO;oBACL,EAAE,EAAE,IAAA,mBAAU,GAAE,CAAC,QAAQ,EAAE;oBAC3B,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,kBAAkB,GAAG,IAAI,CAAC,4BAA4B,CAAC,QAAQ,CAAC,CAAC;YACvE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAEhC,KAAK,IAAI,OAAO,IAAI,kBAAkB,CAAC,yBAAyB,EAAE,CAAC;gBACjE,iBAAiB,CAAC,IAAI,CAAC;oBACrB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,EAAE,EAAE,OAAO,CAAC,EAAE;oBACd,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,SAAS,EAAE,OAAO,CAAC,SAAS;iBAC7B,CAAC,CAAC;gBACH,gBAAgB,CAAC,IAAI,CAAC;oBACpB,EAAE,EAAE,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE;oBACzB,MAAM,EAAE,OAAO,CAAC,SAAS;oBACzB,OAAO,EAAE;wBACP,QAAQ,EAAE,CAAC,QAAQ,CAAC;wBACpB,IAAI,EAAE,OAAO,CAAC,IAAI;qBACnB;iBACF,CAAC,CAAC;YACL,CAAC;YACD,oBAAoB,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;YAEtE,OAAO;gBACL,gBAAgB,EAAE,gBAAgB;gBAClC,SAAS,EAAE,SAAS;gBACpB,oBAAoB,EAAE,oBAAoB;gBAC1C,iBAAiB,EAAE,iBAAiB;aACrC,CAAC;QACJ,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CACX,oEAAoE,EACpE,CAAC,CACF,CAAC;YACF,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAYD;;OAEG;IACH,4BAA4B,CAAC,IAAqB,EAAE,SAAkB;QACpE,MAAM,yBAAyB,GAAoB,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,oBAAoB,GAAU,EAAE,CAAC;QACvC,MAAM,mBAAmB,GAAG,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,IAAI,CAAC,MAAM,CAAC,SAAU,CAAC;QAEhE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEzB,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAElB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,IAAI,OAAO,CAAC,CAAC,CAAC;oBAAE,SAAS;gBAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACxE,IAAI,GAAG,GAAG,mBAAmB,EAAE,CAAC;oBAC9B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBACpB,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;oBAClB,oBAAoB,CAAC,IAAI,CAAC;wBACxB,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;wBACtB,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;wBACtB,UAAU,EAAE,GAAG;qBAChB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,8DAA8D;YAC9D,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,oCAAoC;QAChF,CAAC;QAED,OAAO;YACL,yBAAyB,EAAE,yBAAyB;YACpD,oBAAoB,EAAE,oBAAoB;SAC3C,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,MAA+B;QACpD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACnC,IAAA,wCAA2B,EAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;CACF;AAnID,8CAmIC"}

package/dist/embeddings/embedding_generator.d.ts

@@ -1,30 +0,0 @@
-/**
- * Standalone EmbeddingGenerator module
- * Generates embeddings for concepts and facts
- */
-import { EmbeddingGeneratorConfig } from "../configs/embedding_generator_config";
-export interface ConceptFact {
-    text: string;
-    type: string;
-    reference: string;
-}
-export interface ConceptWithEmbedding {
-    text: string;
-    type: string;
-    reference: string;
-    embedding: number[];
-}
-export declare class EmbeddingGenerator {
-    private config;
-    private openAiService;
-    constructor(config: EmbeddingGeneratorConfig);
-    /**
-     * Generate embeddings for concepts and facts
-     */
-    generateEmbeddings(concepts: ConceptFact[]): Promise<{
-        concepts_facts: ConceptWithEmbedding[];
-        metadata: any;
-    }>;
-    private validateConfig;
-}
-//# sourceMappingURL=embedding_generator.d.ts.map

package/dist/embeddings/embedding_generator.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"embedding_generator.d.ts","sourceRoot":"","sources":["../../src/embeddings/embedding_generator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EACL,wBAAwB,EAEzB,MAAM,uCAAuC,CAAC;AAO/C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,aAAa,CAAgB;gBAEzB,MAAM,EAAE,wBAAwB;IAS5C;;OAEG;IACG,kBAAkB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;QACzD,cAAc,EAAE,oBAAoB,EAAE,CAAC;QACvC,QAAQ,EAAE,GAAG,CAAC;KACf,CAAC;IAsCF,OAAO,CAAC,cAAc;CAKvB"}