only_ever_generator 8.1.0 → 8.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/helper/schema_helper/build_summary_schema.d.ts +2 -0
- package/dist/helper/schema_helper/build_summary_schema.d.ts.map +1 -0
- package/dist/helper/schema_helper/build_summary_schema.js +47 -0
- package/dist/helper/schema_helper/build_summary_schema.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +50 -140
- package/dist/index.js.map +1 -1
- package/dist/typology_gen/summarize.d.ts +28 -0
- package/dist/typology_gen/summarize.d.ts.map +1 -0
- package/dist/typology_gen/summarize.js +136 -0
- package/dist/typology_gen/summarize.js.map +1 -0
- package/package.json +1 -1
- package/src/helper/schema_helper/build_summary_schema.ts +43 -0
- package/src/index.ts +40 -156
- package/src/typology_gen/summarize.ts +162 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build_summary_schema.d.ts","sourceRoot":"","sources":["../../../src/helper/schema_helper/build_summary_schema.ts"],"names":[],"mappings":"AAIA,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,GAAE,MAAwB,EAC9B,MAAM,GAAE,OAAc,gBAmCvB"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.buildSummarySchema = buildSummarySchema;
|
|
13
|
+
const sanitize_strings_1 = require("../../utils/sanitize_strings");
|
|
14
|
+
const mongo_helper_1 = require("../mongo_helper");
|
|
15
|
+
// Function to populate enums in a parsed schema
|
|
16
|
+
function buildSummarySchema(headings_1) {
|
|
17
|
+
return __awaiter(this, arguments, void 0, function* (headings, name = "summary_cards", strict = true) {
|
|
18
|
+
var _a, _b, _c, _d, _e, _f;
|
|
19
|
+
(0, mongo_helper_1.setUp)();
|
|
20
|
+
const objectId = new mongo_helper_1.ObjectId("690ae043116a8c929f3a4cd5");
|
|
21
|
+
const schemaObj = yield mongo_helper_1.database.collection("_prompts").findOne({
|
|
22
|
+
_id: objectId,
|
|
23
|
+
});
|
|
24
|
+
if (!schemaObj) {
|
|
25
|
+
throw new Error("Schema not found");
|
|
26
|
+
}
|
|
27
|
+
const schema = schemaObj.schema;
|
|
28
|
+
if (!schema) {
|
|
29
|
+
throw new Error("Schema not found");
|
|
30
|
+
}
|
|
31
|
+
// Deep clone to avoid mutating the original
|
|
32
|
+
const populatedSchema = JSON.parse(schema);
|
|
33
|
+
// Update name and strict mode
|
|
34
|
+
populatedSchema.name = name;
|
|
35
|
+
populatedSchema.strict = strict;
|
|
36
|
+
// Sanitize headings to remove quotes for OpenAI structured outputs
|
|
37
|
+
const sanitizedStrings = (0, sanitize_strings_1.sanitizeStringsForSchema)(headings);
|
|
38
|
+
// Type-safe enum population for reference field in summary_cards items
|
|
39
|
+
// This populates the enum for cards 2-N (takeaway cards)
|
|
40
|
+
if ((_f = (_e = (_d = (_c = (_b = (_a = populatedSchema.schema) === null || _a === void 0 ? void 0 : _a.properties) === null || _b === void 0 ? void 0 : _b.summary_cards) === null || _c === void 0 ? void 0 : _c.items) === null || _d === void 0 ? void 0 : _d.properties) === null || _e === void 0 ? void 0 : _e.reference) === null || _f === void 0 ? void 0 : _f.enum) {
|
|
41
|
+
populatedSchema.schema.properties.summary_cards.items.properties.reference.enum =
|
|
42
|
+
sanitizedStrings;
|
|
43
|
+
}
|
|
44
|
+
return populatedSchema;
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=build_summary_schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build_summary_schema.js","sourceRoot":"","sources":["../../../src/helper/schema_helper/build_summary_schema.ts"],"names":[],"mappings":";;;;;;;;;;;AAIA,gDAsCC;AA1CD,mEAAwE;AACxE,kDAA4D;AAE5D,gDAAgD;AAChD,SAAsB,kBAAkB;yDACtC,QAAkB,EAClB,OAAe,eAAe,EAC9B,SAAkB,IAAI;;QAEtB,IAAA,oBAAK,GAAE,CAAC;QACR,MAAM,QAAQ,GAAG,IAAI,uBAAQ,CAAC,0BAA0B,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,MAAM,uBAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC;YAC9D,GAAG,EAAE,QAAQ;SACd,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAChC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,4CAA4C;QAC5C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE3C,8BAA8B;QAC9B,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAC5B,eAAe,CAAC,MAAM,GAAG,MAAM,CAAC;QAEhC,mEAAmE;QACnE,MAAM,gBAAgB,GAAG,IAAA,2CAAwB,EAAC,QAAQ,CAAC,CAAC;QAE5D,uEAAuE;QACvE,yDAAyD;QACzD,IACE,MAAA,MAAA,MAAA,MAAA,MAAA,MAAA,eAAe,CAAC,MAAM,0CAAE,UAAU,0CAAE,aAAa,0CAAE,KAAK,0CAAE,UAAU,0CAChE,SAAS,0CAAE,IAAI,EACnB,CAAC;YACD,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC7E,gBAAgB,CAAC;QACrB,CAAC;QAED,OAAO,eAAe,CAAC;IACzB,CAAC;CAAA"}
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAyBpD,OAAO,EAAE,iBAAiB,EAAE,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,147 +1,57 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
2
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
15
|
exports.OnlyEverGenerator = void 0;
|
|
16
|
+
const mongodb_1 = require("mongodb");
|
|
4
17
|
const app_1 = require("./bootstrap/app");
|
|
5
18
|
Object.defineProperty(exports, "OnlyEverGenerator", { enumerable: true, get: function () { return app_1.OnlyEverGenerator; } });
|
|
19
|
+
const open_ai_service_1 = require("./services/open_ai_service");
|
|
20
|
+
const summarize_1 = require("./typology_gen/summarize");
|
|
21
|
+
const mongo_helper_1 = require("./helper/mongo_helper");
|
|
22
|
+
// import { LocalConsolidation } from "./embedding_generation/consolidation/local_consolidation";
|
|
23
|
+
// import { embeddingsResp } from "./typology-parsed-response";
|
|
24
|
+
// import { GlobalConsolidation } from "./embedding_generation/consolidation/global_consolidation";
|
|
25
|
+
// import { ParseEmbeddingResponse } from "./embedding_generation/parse_embedding_response";
|
|
26
|
+
// import { OpenAIHelper } from "./helper/openai_helper";
|
|
27
|
+
// import { GenerateTypology } from "./typology_gen/generate_typology";
|
|
28
|
+
// import { OpenAiService } from "./services/open_ai_service";
|
|
29
|
+
// import { GenerateConceptFacts } from "./typology_gen/generate_concept_facts";
|
|
30
|
+
// import { BaseParamType } from "./types/base_param_type";
|
|
31
|
+
// import dotenv from "dotenv";
|
|
32
|
+
const express_1 = __importDefault(require("express"));
|
|
33
|
+
const app = (0, express_1.default)();
|
|
34
|
+
const port = 3000;
|
|
6
35
|
// dotenv.config();
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
// content: {
|
|
29
|
-
// source_id: source?._id?.toString() || "",
|
|
30
|
-
// title: source?.title || "",
|
|
31
|
-
// type: source?.source_type || "",
|
|
32
|
-
// headings: source?.headings || [],
|
|
33
|
-
// content: source?.content || [],
|
|
34
|
-
// fields: source?.fields || [],
|
|
35
|
-
// taxonomy: source?.learn_value
|
|
36
|
-
// ? {
|
|
37
|
-
// generate_cards: source?.generate_cards || {
|
|
38
|
-
// state: false,
|
|
39
|
-
// reason: "",
|
|
40
|
-
// },
|
|
41
|
-
// concepts_facts: source?.concepts_facts || [],
|
|
42
|
-
// fields: source?.fields || [],
|
|
43
|
-
// learn_value: source?.learn_value || {
|
|
44
|
-
// value: 0,
|
|
45
|
-
// reason: "",
|
|
46
|
-
// bloom_levels: [],
|
|
47
|
-
// bloom_levels_reason: "",
|
|
48
|
-
// },
|
|
49
|
-
// }
|
|
50
|
-
// : undefined,
|
|
51
|
-
// },
|
|
52
|
-
// };
|
|
53
|
-
// const prompts = await getPrompts("text");
|
|
54
|
-
// const bloomInstructions = prompts.bloom_instructions;
|
|
55
|
-
// const cardInstructions = prompts.card_instructions;
|
|
56
|
-
// const cardExamples = prompts.card_examples;
|
|
57
|
-
// console.log("Initializing OnlyEverGenerator");
|
|
58
|
-
// const onlyEverGenerator = new OnlyEverGenerator(
|
|
59
|
-
// process.env.OPEN_AI_KEY as string,
|
|
60
|
-
// "o4-mini",
|
|
61
|
-
// contentForGen,
|
|
62
|
-
// "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
|
|
63
|
-
// "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
|
|
64
|
-
// "pmpt_688118a923e4819098176a13a2f401920d2ea17d881cc6c6",
|
|
65
|
-
// {
|
|
66
|
-
// bloom_instructions: bloomInstructions,
|
|
67
|
-
// card_instructions: cardInstructions,
|
|
68
|
-
// card_examples: cardExamples,
|
|
69
|
-
// },
|
|
70
|
-
// false,
|
|
71
|
-
// 0
|
|
72
|
-
// );
|
|
73
|
-
// const response = await onlyEverGenerator.generate(false, true);
|
|
74
|
-
// res.send(response);
|
|
75
|
-
// } catch (error) {
|
|
76
|
-
// res.status(500).send(error);
|
|
77
|
-
// }
|
|
78
|
-
// });
|
|
79
|
-
// app.listen(port, () => {
|
|
80
|
-
// console.log(`Server is running on port ${port}`);
|
|
81
|
-
// });
|
|
82
|
-
// (async () => {
|
|
83
|
-
// const promptForSummary = {
|
|
84
|
-
// promptId: "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
|
|
85
|
-
// // version: "13",
|
|
86
|
-
// };
|
|
87
|
-
// const promptForConceptFacts = {
|
|
88
|
-
// promptId: "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
|
|
89
|
-
// // version: "6",
|
|
90
|
-
// };
|
|
91
|
-
// setUp();
|
|
92
|
-
// const source = await database.collection("_source").findOne({
|
|
93
|
-
// _id: new ObjectId("68b114cdf23998242ce03235"),
|
|
94
|
-
// });
|
|
95
|
-
// const contentForGen = {
|
|
96
|
-
// prompt: {
|
|
97
|
-
// typology: "",
|
|
98
|
-
// card_generation: "",
|
|
99
|
-
// },
|
|
100
|
-
// content: {
|
|
101
|
-
// source_id: source?._id?.toString() || "",
|
|
102
|
-
// title: source?.title || "",
|
|
103
|
-
// type: source?.source_type || "",
|
|
104
|
-
// headings: source?.headings || [],
|
|
105
|
-
// content: source?.content || [],
|
|
106
|
-
// fields: source?.fields || [],
|
|
107
|
-
// taxonomy: source?.learn_value
|
|
108
|
-
// ? {
|
|
109
|
-
// generate_cards: source?.generate_cards || {
|
|
110
|
-
// state: false,
|
|
111
|
-
// reason: "",
|
|
112
|
-
// },
|
|
113
|
-
// concepts_facts: source?.concepts_facts || [],
|
|
114
|
-
// fields: source?.fields || [],
|
|
115
|
-
// learn_value: source?.learn_value || {
|
|
116
|
-
// value: 0,
|
|
117
|
-
// reason: "",
|
|
118
|
-
// bloom_levels: [],
|
|
119
|
-
// bloom_levels_reason: "",
|
|
120
|
-
// },
|
|
121
|
-
// }
|
|
122
|
-
// : undefined,
|
|
123
|
-
// },
|
|
124
|
-
// };
|
|
125
|
-
// const prompts = await getPrompts("text");
|
|
126
|
-
// const bloomInstructions = prompts.bloom_instructions;
|
|
127
|
-
// const cardInstructions = prompts.card_instructions;
|
|
128
|
-
// const cardExamples = prompts.card_examples;
|
|
129
|
-
// console.log("Initializing OnlyEverGenerator");
|
|
130
|
-
// const onlyEverGenerator = new OnlyEverGenerator(
|
|
131
|
-
// process.env.OPEN_AI_KEY as string,
|
|
132
|
-
// "o4-mini",
|
|
133
|
-
// contentForGen,
|
|
134
|
-
// "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
|
|
135
|
-
// "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
|
|
136
|
-
// "pmpt_688118a923e4819098176a13a2f401920d2ea17d881cc6c6",
|
|
137
|
-
// {
|
|
138
|
-
// bloom_instructions: bloomInstructions,
|
|
139
|
-
// card_instructions: cardInstructions,
|
|
140
|
-
// card_examples: cardExamples,
|
|
141
|
-
// },
|
|
142
|
-
// false
|
|
143
|
-
// );
|
|
144
|
-
// const response = await onlyEverGenerator.generate(true, false);
|
|
145
|
-
// console.log(response);
|
|
146
|
-
// })();
|
|
36
|
+
app.listen(port, () => {
|
|
37
|
+
console.log(`Server is running on port ${port}`);
|
|
38
|
+
});
|
|
39
|
+
app.get("/summarize", (req, res) => __awaiter(void 0, void 0, void 0, function* () {
|
|
40
|
+
(0, mongo_helper_1.setUp)();
|
|
41
|
+
const promptIdForSummaryCards = "pmpt_6902c30a973481968395935cc6dfa78605f50d2b01d4afbd";
|
|
42
|
+
const sourceId = "68b114dac849a25097ed1778";
|
|
43
|
+
const source = yield mongo_helper_1.database
|
|
44
|
+
.collection("_source")
|
|
45
|
+
.findOne({ _id: new mongodb_1.ObjectId(sourceId) });
|
|
46
|
+
const content = {
|
|
47
|
+
title: source === null || source === void 0 ? void 0 : source.title,
|
|
48
|
+
h1_headings: source === null || source === void 0 ? void 0 : source.headings,
|
|
49
|
+
content: source === null || source === void 0 ? void 0 : source.content,
|
|
50
|
+
};
|
|
51
|
+
const openAiService = new open_ai_service_1.OpenAiService(process.env.OPEN_AI_KEY || "", "gpt-5-mini");
|
|
52
|
+
const summarize = new summarize_1.GenerateSummaryCards(openAiService, sourceId, content, "text", promptIdForSummaryCards, false);
|
|
53
|
+
const summaryCards = yield summarize.generate();
|
|
54
|
+
console.log(summaryCards);
|
|
55
|
+
res.json(summaryCards);
|
|
56
|
+
}));
|
|
147
57
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,qCAAmC;AACnC,yCAAoD;AAyB3C,kGAzBA,uBAAiB,OAyBA;AAxB1B,gEAA2D;AAC3D,wDAAgE;AAChE,wDAAwD;AACxD,iGAAiG;AACjG,+DAA+D;AAC/D,mGAAmG;AACnG,4FAA4F;AAC5F,yDAAyD;AACzD,uEAAuE;AACvE,8DAA8D;AAC9D,gFAAgF;AAChF,2DAA2D;AAC3D,+BAA+B;AAC/B,sDAA8B;AAC9B,MAAM,GAAG,GAAG,IAAA,iBAAO,GAAE,CAAC;AACtB,MAAM,IAAI,GAAG,IAAI,CAAC;AAUlB,mBAAmB;AAEnB,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;IACpB,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,EAAE,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,CAAO,GAAG,EAAE,GAAG,EAAE,EAAE;IACvC,IAAA,oBAAK,GAAE,CAAC;IACR,MAAM,uBAAuB,GAC3B,uDAAuD,CAAC;IAC1D,MAAM,QAAQ,GAAG,0BAA0B,CAAC;IAC5C,MAAM,MAAM,GAAG,MAAM,uBAAQ;SAC1B,UAAU,CAAC,SAAS,CAAC;SACrB,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,kBAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG;QACd,KAAK,EAAE,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAK;QACpB,WAAW,EAAE,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,QAAQ;QAC7B,OAAO,EAAE,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO;KACzB,CAAC;IACF,MAAM,aAAa,GAAG,IAAI,+BAAa,CACrC,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,EAC7B,YAAY,CACb,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,gCAAoB,CACxC,aAAa,EACb,QAAQ,EACR,OAAO,EACP,MAAM,EACN,uBAAuB,EACvB,KAAK,CACN,CAAC;IACF,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAC1B,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AACzB,CAAC,CAAA,CAAC,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { OpenAIHelper } from "../helper/openai_helper";
|
|
2
|
+
import { OpenAiService } from "../services/open_ai_service";
|
|
3
|
+
export declare class GenerateSummaryCards {
|
|
4
|
+
openAiService: OpenAiService;
|
|
5
|
+
openAIHelper: OpenAIHelper;
|
|
6
|
+
sourceId: string;
|
|
7
|
+
generationCurriculum: boolean;
|
|
8
|
+
content: {
|
|
9
|
+
title: string;
|
|
10
|
+
h1_headings?: string[];
|
|
11
|
+
timecodes?: string[];
|
|
12
|
+
content: any[];
|
|
13
|
+
};
|
|
14
|
+
type: string;
|
|
15
|
+
promptIdForSummaryCards: string;
|
|
16
|
+
constructor(openAiService: OpenAiService, sourceId: string, content: {
|
|
17
|
+
title: string;
|
|
18
|
+
h1_headings?: string[];
|
|
19
|
+
timecodes?: string[];
|
|
20
|
+
content: any[];
|
|
21
|
+
}, type: string, promptIdForSummaryCards: string, generationCurriculum: boolean);
|
|
22
|
+
generate(): Promise<{
|
|
23
|
+
summary_cards: any;
|
|
24
|
+
metadata: any;
|
|
25
|
+
} | undefined>;
|
|
26
|
+
parseJson(response: any): any;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=summarize.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summarize.d.ts","sourceRoot":"","sources":["../../src/typology_gen/summarize.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAQ5D,qBAAa,oBAAoB;IACxB,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,EAAE,YAAY,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,OAAO,CAAC;IAC9B,OAAO,EAAE;QACd,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,OAAO,EAAE,GAAG,EAAE,CAAC;KAChB,CAAC;IACK,IAAI,EAAE,MAAM,CAAM;IAClB,uBAAuB,EAAE,MAAM,CAAC;gBAErC,aAAa,EAAE,aAAa,EAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,OAAO,EAAE,GAAG,EAAE,CAAC;KAChB,EACD,IAAI,EAAE,MAAM,EACZ,uBAAuB,EAAE,MAAM,EAC/B,oBAAoB,EAAE,OAAO;IAWzB,QAAQ;;;;IAkGd,SAAS,CAAC,QAAQ,EAAE,GAAG;CAkBxB"}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.GenerateSummaryCards = void 0;
|
|
13
|
+
const openai_helper_1 = require("../helper/openai_helper");
|
|
14
|
+
const logger_1 = require("../logger");
|
|
15
|
+
const sanitize_strings_1 = require("../utils/sanitize_strings");
|
|
16
|
+
const build_summary_schema_1 = require("../helper/schema_helper/build_summary_schema");
|
|
17
|
+
class GenerateSummaryCards {
|
|
18
|
+
constructor(openAiService, sourceId, content, type, promptIdForSummaryCards, generationCurriculum) {
|
|
19
|
+
this.type = "";
|
|
20
|
+
this.openAiService = openAiService;
|
|
21
|
+
this.openAIHelper = new openai_helper_1.OpenAIHelper(this.openAiService.api_key);
|
|
22
|
+
this.content = content;
|
|
23
|
+
this.promptIdForSummaryCards = promptIdForSummaryCards;
|
|
24
|
+
this.type = type;
|
|
25
|
+
this.sourceId = sourceId;
|
|
26
|
+
this.generationCurriculum = generationCurriculum;
|
|
27
|
+
}
|
|
28
|
+
generate() {
|
|
29
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
30
|
+
var _a, _b, _c, _d, _e;
|
|
31
|
+
try {
|
|
32
|
+
const headings = this.type === "text"
|
|
33
|
+
? this.content.h1_headings || [""]
|
|
34
|
+
: this.content.timecodes || [""];
|
|
35
|
+
const schema = yield (0, build_summary_schema_1.buildSummarySchema)(headings.length > 0 ? headings : [""], "summary_cards", true);
|
|
36
|
+
const openAiResponse = yield this.openAIHelper.openAI.responses.create({
|
|
37
|
+
prompt: {
|
|
38
|
+
id: this.promptIdForSummaryCards,
|
|
39
|
+
// version: "35",
|
|
40
|
+
variables: {
|
|
41
|
+
heading_type: this.type == "video" ? "timecode" : "h1 heading",
|
|
42
|
+
},
|
|
43
|
+
},
|
|
44
|
+
max_output_tokens: 30000,
|
|
45
|
+
input: [
|
|
46
|
+
{
|
|
47
|
+
role: "user",
|
|
48
|
+
content: [
|
|
49
|
+
{
|
|
50
|
+
type: "input_text",
|
|
51
|
+
text: JSON.stringify(this.content),
|
|
52
|
+
},
|
|
53
|
+
],
|
|
54
|
+
},
|
|
55
|
+
],
|
|
56
|
+
// input: JSON.stringify(this.content),
|
|
57
|
+
store: true,
|
|
58
|
+
text: {
|
|
59
|
+
format: {
|
|
60
|
+
type: "json_schema",
|
|
61
|
+
name: schema.name,
|
|
62
|
+
strict: schema.strict,
|
|
63
|
+
schema: schema.schema,
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
if (!openAiResponse) {
|
|
68
|
+
yield (0, logger_1.log_error)({
|
|
69
|
+
flow: "manual_generation",
|
|
70
|
+
data: openAiResponse,
|
|
71
|
+
error: "empty_openai_response",
|
|
72
|
+
timestamp: new Date(),
|
|
73
|
+
source_id: this.sourceId,
|
|
74
|
+
type: {
|
|
75
|
+
request_type: "concept_fact",
|
|
76
|
+
n: 1,
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
openAiResponse.metadata = {
|
|
82
|
+
req_time: (_a = openAiResponse.created) !== null && _a !== void 0 ? _a : new Date(),
|
|
83
|
+
req_type: {
|
|
84
|
+
type: "breadth",
|
|
85
|
+
sub_type: "summary_cards",
|
|
86
|
+
n: 1,
|
|
87
|
+
},
|
|
88
|
+
req_tokens: (_c = (_b = openAiResponse.usage) === null || _b === void 0 ? void 0 : _b.input_tokens) !== null && _c !== void 0 ? _c : 0,
|
|
89
|
+
res_tokens: (_e = (_d = openAiResponse.usage) === null || _d === void 0 ? void 0 : _d.output_tokens) !== null && _e !== void 0 ? _e : 0,
|
|
90
|
+
prompt: openAiResponse.prompt,
|
|
91
|
+
// prompt_tokens_details: openAiResponse.usage?.,
|
|
92
|
+
model: this.openAiService.model,
|
|
93
|
+
usage: openAiResponse.usage,
|
|
94
|
+
status: "completed",
|
|
95
|
+
};
|
|
96
|
+
const summaryCards = this.parseJson(openAiResponse);
|
|
97
|
+
return {
|
|
98
|
+
summary_cards: summaryCards,
|
|
99
|
+
metadata: openAiResponse.metadata,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
console.log(error);
|
|
104
|
+
yield (0, logger_1.log_error)({
|
|
105
|
+
flow: this.generationCurriculum
|
|
106
|
+
? "curriculum_manual_generation"
|
|
107
|
+
: "manual_generation",
|
|
108
|
+
data: error,
|
|
109
|
+
timestamp: new Date(),
|
|
110
|
+
error: "error_in_concept_facts_generation",
|
|
111
|
+
source_id: this.sourceId,
|
|
112
|
+
type: {
|
|
113
|
+
request_type: "concept_fact",
|
|
114
|
+
n: 1,
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
parseJson(response) {
|
|
122
|
+
try {
|
|
123
|
+
const summaryCards = JSON.parse(response.output_text).summary_cards;
|
|
124
|
+
if (summaryCards) {
|
|
125
|
+
return summaryCards.map((item) => {
|
|
126
|
+
return Object.assign(Object.assign({}, item), { reference: (0, sanitize_strings_1.restoreQuotesInString)(item.reference) });
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
catch (e) {
|
|
131
|
+
throw new logger_1.ParsingError("Something went wrong in parsing the json", response);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
exports.GenerateSummaryCards = GenerateSummaryCards;
|
|
136
|
+
//# sourceMappingURL=summarize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"summarize.js","sourceRoot":"","sources":["../../src/typology_gen/summarize.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,2DAAuD;AAEvD,sCAAiE;AACjE,gEAGmC;AACnC,uFAAkF;AAElF,MAAa,oBAAoB;IAa/B,YACE,aAA4B,EAC5B,QAAgB,EAChB,OAKC,EACD,IAAY,EACZ,uBAA+B,EAC/B,oBAA6B;QAbxB,SAAI,GAAW,EAAE,CAAC;QAevB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,IAAI,4BAAY,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QACjE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;IACnD,CAAC;IAEK,QAAQ;;;YACZ,IAAI,CAAC;gBACH,MAAM,QAAQ,GACZ,IAAI,CAAC,IAAI,KAAK,MAAM;oBAClB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;oBAClC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,CAAC;gBACrC,MAAM,MAAM,GAAG,MAAM,IAAA,yCAAkB,EACrC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EACrC,eAAe,EACf,IAAI,CACL,CAAC;gBACF,MAAM,cAAc,GAClB,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;oBAC9C,MAAM,EAAE;wBACN,EAAE,EAAE,IAAI,CAAC,uBAAuB;wBAChC,iBAAiB;wBACjB,SAAS,EAAE;4BACT,YAAY,EAAE,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY;yBAC/D;qBACF;oBACD,iBAAiB,EAAE,KAAK;oBACxB,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,OAAO,EAAE;gCACP;oCACE,IAAI,EAAE,YAAY;oCAClB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC;iCACnC;6BACF;yBACF;qBACF;oBACD,uCAAuC;oBACvC,KAAK,EAAE,IAAI;oBACX,IAAI,EAAE;wBACJ,MAAM,EAAE;4BACN,IAAI,EAAE,aAAa;4BACnB,IAAI,EAAE,MAAM,CAAC,IAAI;4BACjB,MAAM,EAAE,MAAM,CAAC,MAAM;4BACrB,MAAM,EAAE,MAAM,CAAC,MAAM;yBACtB;qBACF;iBACF,CAAC,CAAC;gBACL,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,MAAM,IAAA,kBAAS,EAAC;wBACd,IAAI,EAAE,mBAAmB;wBACzB,IAAI,EAAE,cAAc;wBACpB,KAAK,EAAE,uBAAuB;wBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE;wBACrB,SAAS,EAAE,IAAI,CAAC,QAAQ;wBACxB,IAAI,EAAE;4BACJ,YAAY,EAAE,cAAc;4BAC5B,CAAC,EAAE,CAAC;yBACL;qBACF,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,cAAc,CAAC,QAAQ,GAAG;oBACxB,QAAQ,EAAE,MAAA,cAAc,CAAC,OAAO,mCAAI,IAAI,IAAI,EAAE;oBAC9C,QAAQ,EAAE;wBACR,IAAI,EAAE,SAAS;wBACf,QAAQ,EAAE,eAAe;wBACzB,CAAC,EAAE,CAAC;qBACL;oBACD,UAAU,EAAE,MAAA,MAAA,cAAc,CAAC,KAAK,0CAAE,YAAY,mCAAI,CAAC;oBACnD,UAAU,EAAE,MAAA,MAAA,cAAc,CAAC,KAAK,0CAAE,aAAa,mCAAI,CAAC;oBACpD,MAAM,EAAE,cAAc,CAAC,MAAM;oBAC7B,iDAAiD;oBACjD,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK;oBAC/B,KAAK,EAAE,cAAc,CAAC,KAAK;oBAC3B,MAAM,EAAE,WAAW;iBACpB,CAAC;gBACF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;gBAEpD,OAAO;oBACL,aAAa,EAAE,YAAY;oBAC3B,QAAQ,EAAE,cAAc,CAAC,QAAQ;iBAClC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,IAAA,kBAAS,EAAC;oBACd,IAAI,EAAE,IAAI,CAAC,oBAAoB;wBAC7B,CAAC,CAAC,8BAA8B;wBAChC,CAAC,CAAC,mBAAmB;oBACvB,IAAI,EAAE,KAAK;oBACX,SAAS,EAAE,IAAI,IAAI,EAAE;oBACrB,KAAK,EAAE,mCAAmC;oBAC1C,SAAS,EAAE,IAAI,CAAC,QAAQ;oBACxB,IAAI,EAAE;wBACJ,YAAY,EAAE,cAAc;wBAC5B,CAAC,EAAE,CAAC;qBACL;iBACF,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;QACH,CAAC;KAAA;IAED,SAAS,CAAC,QAAa;QACrB,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,aAAa,CAAC;YACpE,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE;oBACpC,uCACK,IAAI,KACP,SAAS,EAAE,IAAA,wCAAqB,EAAC,IAAI,CAAC,SAAS,CAAC,IAChD;gBACJ,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,qBAAY,CACpB,0CAA0C,EAC1C,QAAQ,CACT,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAvJD,oDAuJC"}
|
package/package.json
CHANGED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { sanitizeStringsForSchema } from "../../utils/sanitize_strings";
|
|
2
|
+
import { database, ObjectId, setUp } from "../mongo_helper";
|
|
3
|
+
|
|
4
|
+
// Function to populate enums in a parsed schema
|
|
5
|
+
export async function buildSummarySchema(
|
|
6
|
+
headings: string[],
|
|
7
|
+
name: string = "summary_cards",
|
|
8
|
+
strict: boolean = true
|
|
9
|
+
) {
|
|
10
|
+
setUp();
|
|
11
|
+
const objectId = new ObjectId("690ae043116a8c929f3a4cd5");
|
|
12
|
+
const schemaObj = await database.collection("_prompts").findOne({
|
|
13
|
+
_id: objectId,
|
|
14
|
+
});
|
|
15
|
+
if (!schemaObj) {
|
|
16
|
+
throw new Error("Schema not found");
|
|
17
|
+
}
|
|
18
|
+
const schema = schemaObj.schema;
|
|
19
|
+
if (!schema) {
|
|
20
|
+
throw new Error("Schema not found");
|
|
21
|
+
}
|
|
22
|
+
// Deep clone to avoid mutating the original
|
|
23
|
+
const populatedSchema = JSON.parse(schema);
|
|
24
|
+
|
|
25
|
+
// Update name and strict mode
|
|
26
|
+
populatedSchema.name = name;
|
|
27
|
+
populatedSchema.strict = strict;
|
|
28
|
+
|
|
29
|
+
// Sanitize headings to remove quotes for OpenAI structured outputs
|
|
30
|
+
const sanitizedStrings = sanitizeStringsForSchema(headings);
|
|
31
|
+
|
|
32
|
+
// Type-safe enum population for reference field in summary_cards items
|
|
33
|
+
// This populates the enum for cards 2-N (takeaway cards)
|
|
34
|
+
if (
|
|
35
|
+
populatedSchema.schema?.properties?.summary_cards?.items?.properties
|
|
36
|
+
?.reference?.enum
|
|
37
|
+
) {
|
|
38
|
+
populatedSchema.schema.properties.summary_cards.items.properties.reference.enum =
|
|
39
|
+
sanitizedStrings;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return populatedSchema;
|
|
43
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
+
import { ObjectId } from "mongodb";
|
|
1
2
|
import { OnlyEverGenerator } from "./bootstrap/app";
|
|
3
|
+
import { OpenAiService } from "./services/open_ai_service";
|
|
4
|
+
import { GenerateSummaryCards } from "./typology_gen/summarize";
|
|
5
|
+
import { database, setUp } from "./helper/mongo_helper";
|
|
2
6
|
// import { LocalConsolidation } from "./embedding_generation/consolidation/local_consolidation";
|
|
3
7
|
// import { embeddingsResp } from "./typology-parsed-response";
|
|
4
8
|
// import { GlobalConsolidation } from "./embedding_generation/consolidation/global_consolidation";
|
|
@@ -9,9 +13,9 @@ import { OnlyEverGenerator } from "./bootstrap/app";
|
|
|
9
13
|
// import { GenerateConceptFacts } from "./typology_gen/generate_concept_facts";
|
|
10
14
|
// import { BaseParamType } from "./types/base_param_type";
|
|
11
15
|
// import dotenv from "dotenv";
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
16
|
+
import express from "express";
|
|
17
|
+
const app = express();
|
|
18
|
+
const port = 3000;
|
|
15
19
|
|
|
16
20
|
// // import { OnlyEverGenerator } from "./bootstrap/app";
|
|
17
21
|
// import { database, ObjectId, setUp } from "./helper/mongo_helper";
|
|
@@ -23,156 +27,36 @@ import { OnlyEverGenerator } from "./bootstrap/app";
|
|
|
23
27
|
export { OnlyEverGenerator };
|
|
24
28
|
// dotenv.config();
|
|
25
29
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
// generate_cards: source?.generate_cards || {
|
|
60
|
-
// state: false,
|
|
61
|
-
// reason: "",
|
|
62
|
-
// },
|
|
63
|
-
// concepts_facts: source?.concepts_facts || [],
|
|
64
|
-
// fields: source?.fields || [],
|
|
65
|
-
// learn_value: source?.learn_value || {
|
|
66
|
-
// value: 0,
|
|
67
|
-
// reason: "",
|
|
68
|
-
// bloom_levels: [],
|
|
69
|
-
// bloom_levels_reason: "",
|
|
70
|
-
// },
|
|
71
|
-
// }
|
|
72
|
-
// : undefined,
|
|
73
|
-
// },
|
|
74
|
-
// };
|
|
75
|
-
// const prompts = await getPrompts("text");
|
|
76
|
-
// const bloomInstructions = prompts.bloom_instructions;
|
|
77
|
-
// const cardInstructions = prompts.card_instructions;
|
|
78
|
-
// const cardExamples = prompts.card_examples;
|
|
79
|
-
|
|
80
|
-
// console.log("Initializing OnlyEverGenerator");
|
|
81
|
-
// const onlyEverGenerator = new OnlyEverGenerator(
|
|
82
|
-
// process.env.OPEN_AI_KEY as string,
|
|
83
|
-
// "o4-mini",
|
|
84
|
-
// contentForGen,
|
|
85
|
-
|
|
86
|
-
// "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
|
|
87
|
-
// "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
|
|
88
|
-
// "pmpt_688118a923e4819098176a13a2f401920d2ea17d881cc6c6",
|
|
89
|
-
// {
|
|
90
|
-
// bloom_instructions: bloomInstructions,
|
|
91
|
-
// card_instructions: cardInstructions,
|
|
92
|
-
// card_examples: cardExamples,
|
|
93
|
-
// },
|
|
94
|
-
// false,
|
|
95
|
-
// 0
|
|
96
|
-
// );
|
|
97
|
-
// const response = await onlyEverGenerator.generate(false, true);
|
|
98
|
-
|
|
99
|
-
// res.send(response);
|
|
100
|
-
// } catch (error) {
|
|
101
|
-
// res.status(500).send(error);
|
|
102
|
-
// }
|
|
103
|
-
// });
|
|
104
|
-
|
|
105
|
-
// app.listen(port, () => {
|
|
106
|
-
// console.log(`Server is running on port ${port}`);
|
|
107
|
-
// });
|
|
108
|
-
|
|
109
|
-
// (async () => {
|
|
110
|
-
// const promptForSummary = {
|
|
111
|
-
// promptId: "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
|
|
112
|
-
// // version: "13",
|
|
113
|
-
// };
|
|
114
|
-
// const promptForConceptFacts = {
|
|
115
|
-
// promptId: "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
|
|
116
|
-
// // version: "6",
|
|
117
|
-
// };
|
|
118
|
-
|
|
119
|
-
// setUp();
|
|
120
|
-
|
|
121
|
-
// const source = await database.collection("_source").findOne({
|
|
122
|
-
// _id: new ObjectId("68b114cdf23998242ce03235"),
|
|
123
|
-
// });
|
|
124
|
-
|
|
125
|
-
// const contentForGen = {
|
|
126
|
-
// prompt: {
|
|
127
|
-
// typology: "",
|
|
128
|
-
// card_generation: "",
|
|
129
|
-
// },
|
|
130
|
-
// content: {
|
|
131
|
-
// source_id: source?._id?.toString() || "",
|
|
132
|
-
// title: source?.title || "",
|
|
133
|
-
// type: source?.source_type || "",
|
|
134
|
-
// headings: source?.headings || [],
|
|
135
|
-
// content: source?.content || [],
|
|
136
|
-
// fields: source?.fields || [],
|
|
137
|
-
// taxonomy: source?.learn_value
|
|
138
|
-
// ? {
|
|
139
|
-
// generate_cards: source?.generate_cards || {
|
|
140
|
-
// state: false,
|
|
141
|
-
// reason: "",
|
|
142
|
-
// },
|
|
143
|
-
// concepts_facts: source?.concepts_facts || [],
|
|
144
|
-
// fields: source?.fields || [],
|
|
145
|
-
// learn_value: source?.learn_value || {
|
|
146
|
-
// value: 0,
|
|
147
|
-
// reason: "",
|
|
148
|
-
// bloom_levels: [],
|
|
149
|
-
// bloom_levels_reason: "",
|
|
150
|
-
// },
|
|
151
|
-
// }
|
|
152
|
-
// : undefined,
|
|
153
|
-
// },
|
|
154
|
-
// };
|
|
155
|
-
// const prompts = await getPrompts("text");
|
|
156
|
-
// const bloomInstructions = prompts.bloom_instructions;
|
|
157
|
-
// const cardInstructions = prompts.card_instructions;
|
|
158
|
-
// const cardExamples = prompts.card_examples;
|
|
159
|
-
|
|
160
|
-
// console.log("Initializing OnlyEverGenerator");
|
|
161
|
-
// const onlyEverGenerator = new OnlyEverGenerator(
|
|
162
|
-
// process.env.OPEN_AI_KEY as string,
|
|
163
|
-
// "o4-mini",
|
|
164
|
-
// contentForGen,
|
|
165
|
-
|
|
166
|
-
// "pmpt_687a8872d1c8819088a9cdc97dcb688b0893663621695594",
|
|
167
|
-
// "pmpt_687aa547f99c8193b99467ca53630c2607f5a8caf451e7e8",
|
|
168
|
-
// "pmpt_688118a923e4819098176a13a2f401920d2ea17d881cc6c6",
|
|
169
|
-
// {
|
|
170
|
-
// bloom_instructions: bloomInstructions,
|
|
171
|
-
// card_instructions: cardInstructions,
|
|
172
|
-
// card_examples: cardExamples,
|
|
173
|
-
// },
|
|
174
|
-
// false
|
|
175
|
-
// );
|
|
176
|
-
// const response = await onlyEverGenerator.generate(true, false);
|
|
177
|
-
// console.log(response);
|
|
178
|
-
// })();
|
|
30
|
+
app.listen(port, () => {
|
|
31
|
+
console.log(`Server is running on port ${port}`);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
app.get("/summarize", async (req, res) => {
|
|
35
|
+
setUp();
|
|
36
|
+
const promptIdForSummaryCards =
|
|
37
|
+
"pmpt_6902c30a973481968395935cc6dfa78605f50d2b01d4afbd";
|
|
38
|
+
const sourceId = "68b114dac849a25097ed1778";
|
|
39
|
+
const source = await database
|
|
40
|
+
.collection("_source")
|
|
41
|
+
.findOne({ _id: new ObjectId(sourceId) });
|
|
42
|
+
const content = {
|
|
43
|
+
title: source?.title,
|
|
44
|
+
h1_headings: source?.headings,
|
|
45
|
+
content: source?.content,
|
|
46
|
+
};
|
|
47
|
+
const openAiService = new OpenAiService(
|
|
48
|
+
process.env.OPEN_AI_KEY || "",
|
|
49
|
+
"gpt-5-mini"
|
|
50
|
+
);
|
|
51
|
+
const summarize = new GenerateSummaryCards(
|
|
52
|
+
openAiService,
|
|
53
|
+
sourceId,
|
|
54
|
+
content,
|
|
55
|
+
"text",
|
|
56
|
+
promptIdForSummaryCards,
|
|
57
|
+
false
|
|
58
|
+
);
|
|
59
|
+
const summaryCards = await summarize.generate();
|
|
60
|
+
console.log(summaryCards);
|
|
61
|
+
res.json(summaryCards);
|
|
62
|
+
});
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import { buildConceptFactSchema } from "../helper/schema_helper/build_concept_facts_schema";
|
|
2
|
+
import { OpenAIHelper } from "../helper/openai_helper";
|
|
3
|
+
import { OpenAiService } from "../services/open_ai_service";
|
|
4
|
+
import { ErrorLogger, log_error, ParsingError } from "../logger";
|
|
5
|
+
import {
|
|
6
|
+
restoreQuotesInObject,
|
|
7
|
+
restoreQuotesInString,
|
|
8
|
+
} from "../utils/sanitize_strings";
|
|
9
|
+
import { buildSummarySchema } from "../helper/schema_helper/build_summary_schema";
|
|
10
|
+
|
|
11
|
+
export class GenerateSummaryCards {
|
|
12
|
+
public openAiService: OpenAiService;
|
|
13
|
+
public openAIHelper: OpenAIHelper;
|
|
14
|
+
public sourceId: string;
|
|
15
|
+
public generationCurriculum: boolean;
|
|
16
|
+
public content: {
|
|
17
|
+
title: string;
|
|
18
|
+
h1_headings?: string[];
|
|
19
|
+
timecodes?: string[];
|
|
20
|
+
content: any[];
|
|
21
|
+
};
|
|
22
|
+
public type: string = "";
|
|
23
|
+
public promptIdForSummaryCards: string;
|
|
24
|
+
constructor(
|
|
25
|
+
openAiService: OpenAiService,
|
|
26
|
+
sourceId: string,
|
|
27
|
+
content: {
|
|
28
|
+
title: string;
|
|
29
|
+
h1_headings?: string[];
|
|
30
|
+
timecodes?: string[];
|
|
31
|
+
content: any[];
|
|
32
|
+
},
|
|
33
|
+
type: string,
|
|
34
|
+
promptIdForSummaryCards: string,
|
|
35
|
+
generationCurriculum: boolean
|
|
36
|
+
) {
|
|
37
|
+
this.openAiService = openAiService;
|
|
38
|
+
this.openAIHelper = new OpenAIHelper(this.openAiService.api_key);
|
|
39
|
+
this.content = content;
|
|
40
|
+
this.promptIdForSummaryCards = promptIdForSummaryCards;
|
|
41
|
+
this.type = type;
|
|
42
|
+
this.sourceId = sourceId;
|
|
43
|
+
this.generationCurriculum = generationCurriculum;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async generate() {
|
|
47
|
+
try {
|
|
48
|
+
const headings =
|
|
49
|
+
this.type === "text"
|
|
50
|
+
? this.content.h1_headings || [""]
|
|
51
|
+
: this.content.timecodes || [""];
|
|
52
|
+
const schema = await buildSummarySchema(
|
|
53
|
+
headings.length > 0 ? headings : [""],
|
|
54
|
+
"summary_cards",
|
|
55
|
+
true
|
|
56
|
+
);
|
|
57
|
+
const openAiResponse: any =
|
|
58
|
+
await this.openAIHelper.openAI.responses.create({
|
|
59
|
+
prompt: {
|
|
60
|
+
id: this.promptIdForSummaryCards,
|
|
61
|
+
// version: "35",
|
|
62
|
+
variables: {
|
|
63
|
+
heading_type: this.type == "video" ? "timecode" : "h1 heading",
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
max_output_tokens: 30000,
|
|
67
|
+
input: [
|
|
68
|
+
{
|
|
69
|
+
role: "user",
|
|
70
|
+
content: [
|
|
71
|
+
{
|
|
72
|
+
type: "input_text",
|
|
73
|
+
text: JSON.stringify(this.content),
|
|
74
|
+
},
|
|
75
|
+
],
|
|
76
|
+
},
|
|
77
|
+
],
|
|
78
|
+
// input: JSON.stringify(this.content),
|
|
79
|
+
store: true,
|
|
80
|
+
text: {
|
|
81
|
+
format: {
|
|
82
|
+
type: "json_schema",
|
|
83
|
+
name: schema.name,
|
|
84
|
+
strict: schema.strict,
|
|
85
|
+
schema: schema.schema,
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
if (!openAiResponse) {
|
|
90
|
+
await log_error({
|
|
91
|
+
flow: "manual_generation",
|
|
92
|
+
data: openAiResponse,
|
|
93
|
+
error: "empty_openai_response",
|
|
94
|
+
timestamp: new Date(),
|
|
95
|
+
source_id: this.sourceId,
|
|
96
|
+
type: {
|
|
97
|
+
request_type: "concept_fact",
|
|
98
|
+
n: 1,
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
openAiResponse.metadata = {
|
|
105
|
+
req_time: openAiResponse.created ?? new Date(),
|
|
106
|
+
req_type: {
|
|
107
|
+
type: "breadth",
|
|
108
|
+
sub_type: "summary_cards",
|
|
109
|
+
n: 1,
|
|
110
|
+
},
|
|
111
|
+
req_tokens: openAiResponse.usage?.input_tokens ?? 0,
|
|
112
|
+
res_tokens: openAiResponse.usage?.output_tokens ?? 0,
|
|
113
|
+
prompt: openAiResponse.prompt,
|
|
114
|
+
// prompt_tokens_details: openAiResponse.usage?.,
|
|
115
|
+
model: this.openAiService.model,
|
|
116
|
+
usage: openAiResponse.usage,
|
|
117
|
+
status: "completed",
|
|
118
|
+
};
|
|
119
|
+
const summaryCards = this.parseJson(openAiResponse);
|
|
120
|
+
|
|
121
|
+
return {
|
|
122
|
+
summary_cards: summaryCards,
|
|
123
|
+
metadata: openAiResponse.metadata,
|
|
124
|
+
};
|
|
125
|
+
} catch (error) {
|
|
126
|
+
console.log(error);
|
|
127
|
+
await log_error({
|
|
128
|
+
flow: this.generationCurriculum
|
|
129
|
+
? "curriculum_manual_generation"
|
|
130
|
+
: "manual_generation",
|
|
131
|
+
data: error,
|
|
132
|
+
timestamp: new Date(),
|
|
133
|
+
error: "error_in_concept_facts_generation",
|
|
134
|
+
source_id: this.sourceId,
|
|
135
|
+
type: {
|
|
136
|
+
request_type: "concept_fact",
|
|
137
|
+
n: 1,
|
|
138
|
+
},
|
|
139
|
+
});
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
parseJson(response: any) {
|
|
145
|
+
try {
|
|
146
|
+
const summaryCards = JSON.parse(response.output_text).summary_cards;
|
|
147
|
+
if (summaryCards) {
|
|
148
|
+
return summaryCards.map((item: any) => {
|
|
149
|
+
return {
|
|
150
|
+
...item,
|
|
151
|
+
reference: restoreQuotesInString(item.reference),
|
|
152
|
+
};
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
} catch (e) {
|
|
156
|
+
throw new ParsingError(
|
|
157
|
+
"Something went wrong in parsing the json",
|
|
158
|
+
response
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|