only_ever_generator 8.4.6 → 8.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bootstrap/app.d.ts +6 -3
- package/dist/bootstrap/app.d.ts.map +1 -1
- package/dist/bootstrap/app.js +11 -17
- package/dist/bootstrap/app.js.map +1 -1
- package/dist/card_gen/generate_cards.d.ts +2 -3
- package/dist/card_gen/generate_cards.d.ts.map +1 -1
- package/dist/card_gen/generate_cards.js +22 -15
- package/dist/card_gen/generate_cards.js.map +1 -1
- package/dist/constants/prompt_data.d.ts +4 -4
- package/dist/constants/prompt_data.js +302 -302
- package/dist/constants/prompts/card_gen_prompt.js +160 -160
- package/dist/constants/prompts/typology_prompt.js +131 -131
- package/dist/constants/source_data.d.ts +171 -171
- package/dist/constants/source_data.js +973 -973
- package/dist/embedding_generation/local_consolidation.js +104 -104
- package/dist/helper/build_concept_facts_schema.d.ts +42 -42
- package/dist/helper/build_concept_facts_schema.js +44 -44
- package/dist/helper/qdrant_db_methods.d.ts.map +1 -1
- package/dist/helper/schema_helper/build_card_schema.d.ts +1 -9
- package/dist/helper/schema_helper/build_card_schema.d.ts.map +1 -1
- package/dist/helper/schema_helper/build_card_schema.js +47 -50
- package/dist/helper/schema_helper/build_card_schema.js.map +1 -1
- package/dist/helper/schema_helper/build_concept_facts_schema.d.ts +1 -1
- package/dist/helper/schema_helper/build_concept_facts_schema.d.ts.map +1 -1
- package/dist/helper/schema_helper/build_concept_facts_schema.js +20 -5
- package/dist/helper/schema_helper/build_concept_facts_schema.js.map +1 -1
- package/dist/helper/schema_helper/build_summary_schema.d.ts +1 -1
- package/dist/helper/schema_helper/build_summary_schema.d.ts.map +1 -1
- package/dist/helper/schema_helper/build_summary_schema.js +18 -7
- package/dist/helper/schema_helper/build_summary_schema.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/parse/response_format_card.d.ts +176 -176
- package/dist/parse/response_format_card.js +371 -371
- package/dist/parse/response_format_typology.d.ts +1 -1
- package/dist/parse/response_format_typology.js +46 -46
- package/dist/services/get_prompts.d.ts +8 -7
- package/dist/services/get_prompts.d.ts.map +1 -1
- package/dist/services/get_prompts.js +69 -21
- package/dist/services/get_prompts.js.map +1 -1
- package/dist/typology_gen/generate_concept_facts.d.ts +2 -3
- package/dist/typology_gen/generate_concept_facts.d.ts.map +1 -1
- package/dist/typology_gen/generate_concept_facts.js +25 -15
- package/dist/typology_gen/generate_concept_facts.js.map +1 -1
- package/dist/typology_gen/generate_typology.d.ts +2 -1
- package/dist/typology_gen/generate_typology.d.ts.map +1 -1
- package/dist/typology_gen/generate_typology.js +24 -13
- package/dist/typology_gen/generate_typology.js.map +1 -1
- package/dist/typology_gen/summarize.d.ts +2 -3
- package/dist/typology_gen/summarize.d.ts.map +1 -1
- package/dist/typology_gen/summarize.js +24 -13
- package/dist/typology_gen/summarize.js.map +1 -1
- package/package.json +39 -38
- package/src/bootstrap/app.ts +418 -416
- package/src/card_gen/generate_cards.ts +347 -345
- package/src/config.ts +11 -11
- package/src/constants/api_constants.ts +7 -7
- package/src/constants/prompts/card_gen_prompt.ts +164 -164
- package/src/constants/prompts/typology_prompt.ts +139 -139
- package/src/embedding_generation/consolidation/global_consolidation.ts +96 -96
- package/src/embedding_generation/consolidation/local_consolidation.ts +141 -141
- package/src/embedding_generation/consolidation/write_consolidated_data.ts +98 -98
- package/src/embedding_generation/generate_embeddings.ts +42 -42
- package/src/embedding_generation/parse_embedding_response.ts +31 -31
- package/src/enums/card_type_enum.ts +6 -6
- package/src/gap_fill/calculate_gap_fill.ts +50 -50
- package/src/helper/get_id_from_title.ts +33 -33
- package/src/helper/mongo_helper.ts +29 -29
- package/src/helper/openai_helper.ts +20 -20
- package/src/helper/qdrant_db_methods.ts +77 -77
- package/src/helper/schema_helper/build_card_schema.ts +74 -98
- package/src/helper/schema_helper/build_classify_summarize_schema.ts +43 -43
- package/src/helper/schema_helper/build_concept_facts_schema.ts +45 -31
- package/src/helper/schema_helper/build_summary_schema.ts +43 -32
- package/src/index.ts +71 -73
- package/src/logger.ts +65 -65
- package/src/parse/parse_card/parse_cloze_card.ts +146 -146
- package/src/parse/parse_card/parse_flash_cards.ts +42 -42
- package/src/parse/parse_card/parse_match_card.ts +104 -104
- package/src/parse/parse_card/parse_mcq_card.ts +114 -114
- package/src/parse/parse_card_response.ts +197 -197
- package/src/parse/parse_source_content.ts +212 -212
- package/src/services/get_prompts.ts +164 -112
- package/src/services/open_ai_service.ts +89 -89
- package/src/services/qdrant_service.ts +10 -10
- package/src/types/base_param_type.ts +13 -13
- package/src/types/mongo_concept_fact_type.ts +12 -12
- package/src/types/parsed_card_type.ts +39 -39
- package/src/types/raw_card_response_types/generated_card_response_type.ts +59 -59
- package/src/types/source_taxonomy_type.ts +24 -24
- package/src/typology-parsed-response.ts +1932 -1932
- package/src/typology_gen/generate_concept_facts.ts +180 -169
- package/src/typology_gen/generate_typology.ts +203 -189
- package/src/typology_gen/summarize.ts +176 -164
- package/src/utils/distributed_quote_restoration.ts +80 -80
- package/src/utils/generate_args.ts +29 -29
- package/src/utils/parse_openai_response.ts +19 -19
- package/src/utils/sanitize_strings.ts +65 -65
- package/tsconfig.json +16 -16
- package/dist/constants/default_generation_variables.d.ts +0 -3
- package/dist/constants/default_generation_variables.d.ts.map +0 -1
- package/dist/constants/default_generation_variables.js +0 -580
- package/dist/constants/default_generation_variables.js.map +0 -1
- package/dist/services/prompts_test.d.ts +0 -10
- package/dist/services/prompts_test.d.ts.map +0 -1
- package/dist/services/prompts_test.js +0 -227
- package/dist/services/prompts_test.js.map +0 -1
- package/dist/types/generation_variables_schema.d.ts +0 -14
- package/dist/types/generation_variables_schema.d.ts.map +0 -1
- package/dist/types/generation_variables_schema.js +0 -3
- package/dist/types/generation_variables_schema.js.map +0 -1
- package/dist/utils/test.d.ts +0 -2
- package/dist/utils/test.d.ts.map +0 -1
- package/dist/utils/test.js +0 -5
- package/dist/utils/test.js.map +0 -1
- package/src/constants/default_generation_variables.ts +0 -624
- package/src/types/generation_variables_schema.ts +0 -16
|
@@ -1,65 +1,65 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Utility functions for sanitizing strings for OpenAI structured outputs
|
|
3
|
-
* and restoring them back to their original form without requiring original mapping
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
// Use a unique, reversible encoding for quotes
|
|
7
|
-
const QUOTE_PLACEHOLDER = "___QUOTE___";
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Sanitizes an array of strings by replacing quotes with a safe placeholder
|
|
11
|
-
* This approach is stateless and doesn't require storing mappings
|
|
12
|
-
* @param strings - Array of strings to sanitize
|
|
13
|
-
* @returns Array of sanitized strings
|
|
14
|
-
*/
|
|
15
|
-
export function sanitizeStringsForSchema(strings: string[]): string[] {
|
|
16
|
-
return strings.map((str) => str.replace(/"/g, QUOTE_PLACEHOLDER));
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
/**
|
|
20
|
-
* Restores quotes in a string by replacing placeholders with actual quotes
|
|
21
|
-
* This works without needing the original mapping
|
|
22
|
-
* @param sanitizedString - String with quote placeholders
|
|
23
|
-
* @returns Original string with quotes restored
|
|
24
|
-
*/
|
|
25
|
-
export function restoreQuotesInString(sanitizedString: string): string {
|
|
26
|
-
return sanitizedString.replace(new RegExp(QUOTE_PLACEHOLDER, "g"), '"');
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Recursively restores quotes in an object's string values
|
|
31
|
-
* This works across Lambda functions without requiring original context
|
|
32
|
-
* @param obj - Object to process
|
|
33
|
-
* @returns Object with quotes restored
|
|
34
|
-
*/
|
|
35
|
-
export function restoreQuotesInObject(obj: any): any {
|
|
36
|
-
if (typeof obj === "string") {
|
|
37
|
-
return restoreQuotesInString(obj);
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
if (Array.isArray(obj)) {
|
|
41
|
-
return obj.map((item) => restoreQuotesInObject(item));
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
if (obj && typeof obj === "object") {
|
|
45
|
-
const result: any = {};
|
|
46
|
-
for (const [key, value] of Object.entries(obj)) {
|
|
47
|
-
result[key] = restoreQuotesInObject(value);
|
|
48
|
-
}
|
|
49
|
-
return result;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
return obj;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
/**
|
|
56
|
-
* Legacy function signature for backward compatibility
|
|
57
|
-
* Now ignores the originalMapping parameter since it's not needed
|
|
58
|
-
* @deprecated Use restoreQuotesInObject(obj) instead
|
|
59
|
-
*/
|
|
60
|
-
export function restoreQuotesInObjectLegacy(
|
|
61
|
-
obj: any,
|
|
62
|
-
originalMapping?: Map<string, string>
|
|
63
|
-
): any {
|
|
64
|
-
return restoreQuotesInObject(obj);
|
|
65
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Utility functions for sanitizing strings for OpenAI structured outputs
|
|
3
|
+
* and restoring them back to their original form without requiring original mapping
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
// Use a unique, reversible encoding for quotes
|
|
7
|
+
const QUOTE_PLACEHOLDER = "___QUOTE___";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Sanitizes an array of strings by replacing quotes with a safe placeholder
|
|
11
|
+
* This approach is stateless and doesn't require storing mappings
|
|
12
|
+
* @param strings - Array of strings to sanitize
|
|
13
|
+
* @returns Array of sanitized strings
|
|
14
|
+
*/
|
|
15
|
+
export function sanitizeStringsForSchema(strings: string[]): string[] {
|
|
16
|
+
return strings.map((str) => str.replace(/"/g, QUOTE_PLACEHOLDER));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Restores quotes in a string by replacing placeholders with actual quotes
|
|
21
|
+
* This works without needing the original mapping
|
|
22
|
+
* @param sanitizedString - String with quote placeholders
|
|
23
|
+
* @returns Original string with quotes restored
|
|
24
|
+
*/
|
|
25
|
+
export function restoreQuotesInString(sanitizedString: string): string {
|
|
26
|
+
return sanitizedString.replace(new RegExp(QUOTE_PLACEHOLDER, "g"), '"');
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Recursively restores quotes in an object's string values
|
|
31
|
+
* This works across Lambda functions without requiring original context
|
|
32
|
+
* @param obj - Object to process
|
|
33
|
+
* @returns Object with quotes restored
|
|
34
|
+
*/
|
|
35
|
+
export function restoreQuotesInObject(obj: any): any {
|
|
36
|
+
if (typeof obj === "string") {
|
|
37
|
+
return restoreQuotesInString(obj);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (Array.isArray(obj)) {
|
|
41
|
+
return obj.map((item) => restoreQuotesInObject(item));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
if (obj && typeof obj === "object") {
|
|
45
|
+
const result: any = {};
|
|
46
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
47
|
+
result[key] = restoreQuotesInObject(value);
|
|
48
|
+
}
|
|
49
|
+
return result;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return obj;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Legacy function signature for backward compatibility
|
|
57
|
+
* Now ignores the originalMapping parameter since it's not needed
|
|
58
|
+
* @deprecated Use restoreQuotesInObject(obj) instead
|
|
59
|
+
*/
|
|
60
|
+
export function restoreQuotesInObjectLegacy(
|
|
61
|
+
obj: any,
|
|
62
|
+
originalMapping?: Map<string, string>
|
|
63
|
+
): any {
|
|
64
|
+
return restoreQuotesInObject(obj);
|
|
65
|
+
}
|
package/tsconfig.json
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
|
|
4
|
-
"module": "commonjs" /* Specify what module code is generated. */,
|
|
5
|
-
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
|
|
6
|
-
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
|
|
7
|
-
"strict": true,
|
|
8
|
-
"declaration": true,
|
|
9
|
-
"declarationMap": true,
|
|
10
|
-
"emitDeclarationOnly": false,
|
|
11
|
-
"rootDir": "./src",
|
|
12
|
-
"outDir": "./dist", /* Enable all strict type-checking options. */
|
|
13
|
-
"sourceMap": true
|
|
14
|
-
},
|
|
15
|
-
"include": ["src/**/*"],
|
|
16
|
-
"exclude": ["node_modules", "**/*.test.ts"]
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
|
|
4
|
+
"module": "commonjs" /* Specify what module code is generated. */,
|
|
5
|
+
"esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
|
|
6
|
+
"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
|
|
7
|
+
"strict": true,
|
|
8
|
+
"declaration": true,
|
|
9
|
+
"declarationMap": true,
|
|
10
|
+
"emitDeclarationOnly": false,
|
|
11
|
+
"rootDir": "./src",
|
|
12
|
+
"outDir": "./dist", /* Enable all strict type-checking options. */
|
|
13
|
+
"sourceMap": true
|
|
14
|
+
},
|
|
15
|
+
"include": ["src/**/*"],
|
|
16
|
+
"exclude": ["node_modules", "**/*.test.ts"]
|
|
17
17
|
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"default_generation_variables.d.ts","sourceRoot":"","sources":["../../src/constants/default_generation_variables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAEjF,eAAO,MAAM,0BAA0B,EAAE,yBA6mBxC,CAAC"}
|