@zodic/shared 0.0.195 โ 0.0.197
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/app/services/ConceptService.ts +517 -101
- package/package.json +1 -1
- package/utils/buildMessages.ts +44 -1
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
KVNamespaceListKey,
|
|
3
|
+
KVNamespaceListResult,
|
|
4
|
+
} from '@cloudflare/workers-types';
|
|
1
5
|
import { and, eq } from 'drizzle-orm';
|
|
2
6
|
import { inject, injectable } from 'inversify';
|
|
3
7
|
import 'reflect-metadata';
|
|
@@ -28,111 +32,116 @@ export class ConceptService {
|
|
|
28
32
|
console.log(
|
|
29
33
|
`๐ Generating basic info for concept: ${conceptSlug}, combination: ${combinationString}, override: ${override}`
|
|
30
34
|
);
|
|
31
|
-
|
|
35
|
+
|
|
32
36
|
const kvStore = this.context.kvConceptsStore();
|
|
33
37
|
const kvFailuresStore = this.context.kvConceptFailuresStore();
|
|
34
38
|
const kvKeyEN = buildConceptKVKey('en-us', conceptSlug, combinationString);
|
|
35
39
|
const kvKeyPT = buildConceptKVKey('pt-br', conceptSlug, combinationString);
|
|
36
|
-
|
|
37
|
-
|
|
40
|
+
|
|
38
41
|
// โ
Use Durable Object stub
|
|
39
42
|
const id = this.context.env.CONCEPT_NAMES_DO.idFromName(conceptSlug);
|
|
40
43
|
const stub = this.context.env.CONCEPT_NAMES_DO.get(id);
|
|
41
|
-
|
|
44
|
+
|
|
42
45
|
console.log(`๐ก Fetching existing KV data for ${conceptSlug}...`);
|
|
43
46
|
const existingEN = await this.getKVConcept(kvKeyEN);
|
|
44
47
|
const existingPT = await this.getKVConcept(kvKeyPT);
|
|
45
|
-
|
|
48
|
+
|
|
46
49
|
// โ
IMMEDIATE SKIP if already exists and override is false
|
|
47
50
|
if (!override && existingEN.name && existingPT.name) {
|
|
48
51
|
console.log(`โก Basic info already exists for ${conceptSlug}, skipping.`);
|
|
49
|
-
|
|
50
|
-
// โ
**Backfill Durable Object** to register the names
|
|
51
|
-
console.log(`๐ก Backfilling Durable Object for ${conceptSlug}...`);
|
|
52
|
-
try {
|
|
53
|
-
await Promise.all([
|
|
54
|
-
stub.fetch(`https://internal/add-name`, {
|
|
55
|
-
method: 'POST',
|
|
56
|
-
body: JSON.stringify({ language: 'en-us', name: existingEN.name }),
|
|
57
|
-
headers: { 'Content-Type': 'application/json' },
|
|
58
|
-
}),
|
|
59
|
-
stub.fetch(`https://internal/add-name`, {
|
|
60
|
-
method: 'POST',
|
|
61
|
-
body: JSON.stringify({ language: 'pt-br', name: existingPT.name }),
|
|
62
|
-
headers: { 'Content-Type': 'application/json' },
|
|
63
|
-
}),
|
|
64
|
-
]);
|
|
65
|
-
console.log(`โ
Successfully backfilled Durable Object for ${conceptSlug}`);
|
|
66
|
-
} catch (error) {
|
|
67
|
-
console.error(`โ Error backfilling Durable Object:`, error);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
return; // โ
Exit early
|
|
52
|
+
return;
|
|
71
53
|
}
|
|
72
|
-
|
|
54
|
+
|
|
73
55
|
let attempts = 0;
|
|
74
|
-
const maxAttempts =
|
|
75
|
-
|
|
76
|
-
|
|
56
|
+
const maxAttempts = 3;
|
|
57
|
+
|
|
77
58
|
while (attempts < maxAttempts) {
|
|
78
59
|
let phase = 'generation';
|
|
79
60
|
try {
|
|
80
61
|
attempts++;
|
|
81
|
-
console.log(
|
|
82
|
-
|
|
62
|
+
console.log(
|
|
63
|
+
`๐ Attempt ${attempts} to generate basic info for ${conceptSlug}...`
|
|
64
|
+
);
|
|
65
|
+
|
|
83
66
|
// โ
Fetch the latest name list from Durable Object
|
|
84
|
-
console.log(
|
|
67
|
+
console.log(
|
|
68
|
+
`๐ก Fetching names from Durable Object for ${conceptSlug}...`
|
|
69
|
+
);
|
|
85
70
|
let allNamesEN: string[] = [];
|
|
86
71
|
let allNamesPT: string[] = [];
|
|
87
72
|
const response = await stub.fetch(`https://internal/names`);
|
|
88
|
-
|
|
73
|
+
|
|
89
74
|
if (response.ok) {
|
|
90
|
-
const data = (await response.json()) as {
|
|
75
|
+
const data = (await response.json()) as {
|
|
76
|
+
'en-us': string[];
|
|
77
|
+
'pt-br': string[];
|
|
78
|
+
};
|
|
91
79
|
allNamesEN = data['en-us'] || [];
|
|
92
80
|
allNamesPT = data['pt-br'] || [];
|
|
93
|
-
console.log(`โ
Retrieved ${allNamesEN.length} EN names and ${allNamesPT.length} PT names.`);
|
|
94
|
-
} else {
|
|
95
|
-
console.log(`โ Error fetching names from DO for ${conceptSlug}: ${await response.text()}`);
|
|
96
81
|
}
|
|
97
|
-
|
|
82
|
+
|
|
98
83
|
console.log(`โ๏ธ Generating new name...`);
|
|
99
|
-
// โ
Merge newly generated names with existing recent names
|
|
100
|
-
const recentNamesEN = [...allNamesEN.slice(-40), ...newlyGeneratedNames];
|
|
101
|
-
|
|
102
84
|
const messages = this.context
|
|
103
85
|
.buildLLMMessages()
|
|
104
86
|
.generateConceptBasicInfo({
|
|
105
87
|
combination: combinationString,
|
|
106
88
|
conceptSlug,
|
|
107
|
-
existingNames:
|
|
89
|
+
existingNames: allNamesEN.slice(-40),
|
|
108
90
|
});
|
|
109
|
-
|
|
110
|
-
let aiResponse = await this.context
|
|
91
|
+
|
|
92
|
+
let aiResponse = await this.context
|
|
93
|
+
.api()
|
|
94
|
+
.callTogether.single(messages, {});
|
|
111
95
|
if (!aiResponse) throw new Error(`โ AI returned an empty response`);
|
|
112
|
-
|
|
96
|
+
|
|
113
97
|
phase = 'cleaning';
|
|
114
|
-
console.log(`๐งผ Cleaning AI response...`);
|
|
115
98
|
aiResponse = this.cleanAIResponse(aiResponse);
|
|
116
|
-
|
|
99
|
+
|
|
117
100
|
phase = 'parsing';
|
|
118
|
-
console.log(`๐ Parsing AI response...`);
|
|
119
101
|
let { nameEN, descriptionEN, poemEN, namePT, descriptionPT, poemPT } =
|
|
120
102
|
this.parseBasicInfoResponse(aiResponse, conceptSlug);
|
|
121
|
-
|
|
103
|
+
|
|
122
104
|
console.log(`๐ญ Generated names: EN - "${nameEN}", PT - "${namePT}"`);
|
|
123
|
-
|
|
124
|
-
// โ
**
|
|
125
|
-
|
|
126
|
-
|
|
105
|
+
|
|
106
|
+
// โ
**Check name length and retry if too long**
|
|
107
|
+
const MAX_NAME_LENGTH = 44; // Adjust if needed
|
|
108
|
+
if (
|
|
109
|
+
nameEN.length > MAX_NAME_LENGTH ||
|
|
110
|
+
namePT.length > MAX_NAME_LENGTH
|
|
111
|
+
) {
|
|
112
|
+
console.warn(
|
|
113
|
+
`โ ๏ธ Name too long: "${nameEN}" | "${namePT}". Retrying...`
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
// โ
**Register name in Durable Object as blocked**
|
|
117
|
+
await stub.fetch(`https://internal/add-name`, {
|
|
118
|
+
method: 'POST',
|
|
119
|
+
body: JSON.stringify({ language: 'en-us', name: nameEN }),
|
|
120
|
+
headers: { 'Content-Type': 'application/json' },
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
await stub.fetch(`https://internal/add-name`, {
|
|
124
|
+
method: 'POST',
|
|
125
|
+
body: JSON.stringify({ language: 'pt-br', name: namePT }),
|
|
126
|
+
headers: { 'Content-Type': 'application/json' },
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
console.log(`โ
Added long name to DO blocklist, retrying...`);
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
|
|
127
133
|
// โ
Check uniqueness before storing
|
|
128
134
|
if (allNamesEN.includes(nameEN) || allNamesPT.includes(namePT)) {
|
|
129
|
-
console.warn(
|
|
130
|
-
|
|
135
|
+
console.warn(
|
|
136
|
+
`โ ๏ธ Duplicate Name Detected: "${nameEN}" or "${namePT}"`
|
|
137
|
+
);
|
|
138
|
+
|
|
131
139
|
// โ
**On third attempt, store the name anyway & register in kvFailures**
|
|
132
140
|
if (attempts >= maxAttempts) {
|
|
133
|
-
console.log(
|
|
134
|
-
|
|
135
|
-
|
|
141
|
+
console.log(
|
|
142
|
+
`๐จ Max attempts reached. Storing name despite duplicate.`
|
|
143
|
+
);
|
|
144
|
+
|
|
136
145
|
await kvFailuresStore.put(
|
|
137
146
|
`failures:duplicates:${conceptSlug}:${combinationString}`,
|
|
138
147
|
JSON.stringify({
|
|
@@ -144,45 +153,17 @@ export class ConceptService {
|
|
|
144
153
|
timestamp: new Date().toISOString(),
|
|
145
154
|
})
|
|
146
155
|
);
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
headers: { 'Content-Type': 'application/json' },
|
|
153
|
-
});
|
|
154
|
-
|
|
155
|
-
await stub.fetch(`https://internal/add-name`, {
|
|
156
|
-
method: 'POST',
|
|
157
|
-
body: JSON.stringify({ language: 'pt-br', name: namePT }),
|
|
158
|
-
headers: { 'Content-Type': 'application/json' },
|
|
159
|
-
});
|
|
160
|
-
|
|
161
|
-
// โ
Store the generated basic info in KV
|
|
162
|
-
Object.assign(existingEN, {
|
|
163
|
-
name: nameEN,
|
|
164
|
-
description: descriptionEN,
|
|
165
|
-
poem: poemEN,
|
|
166
|
-
status: 'forced', // ๐ฅ Mark as "forced" due to duplication
|
|
167
|
-
});
|
|
168
|
-
await kvStore.put(kvKeyEN, JSON.stringify(existingEN));
|
|
169
|
-
|
|
170
|
-
Object.assign(existingPT, {
|
|
171
|
-
name: namePT,
|
|
172
|
-
description: descriptionPT,
|
|
173
|
-
poem: poemPT,
|
|
174
|
-
status: 'forced', // ๐ฅ Mark as "forced" due to duplication
|
|
175
|
-
});
|
|
176
|
-
await kvStore.put(kvKeyPT, JSON.stringify(existingPT));
|
|
177
|
-
|
|
178
|
-
console.log(`โ
Stored duplicate name after max retries: ${nameEN}, ${namePT}`);
|
|
179
|
-
return;
|
|
156
|
+
|
|
157
|
+
console.log(
|
|
158
|
+
`โ
Stored duplicate name after max retries: ${nameEN}, ${namePT}`
|
|
159
|
+
);
|
|
160
|
+
break;
|
|
180
161
|
}
|
|
181
|
-
|
|
162
|
+
|
|
182
163
|
console.log('๐ Retrying due to duplicate name...');
|
|
183
164
|
continue;
|
|
184
165
|
}
|
|
185
|
-
|
|
166
|
+
|
|
186
167
|
console.log(`๐ Storing names in Durable Object...`);
|
|
187
168
|
// โ
**Immediately update Durable Object with the new name**
|
|
188
169
|
await stub.fetch(`https://internal/add-name`, {
|
|
@@ -190,13 +171,13 @@ export class ConceptService {
|
|
|
190
171
|
body: JSON.stringify({ language: 'en-us', name: nameEN }),
|
|
191
172
|
headers: { 'Content-Type': 'application/json' },
|
|
192
173
|
});
|
|
193
|
-
|
|
174
|
+
|
|
194
175
|
await stub.fetch(`https://internal/add-name`, {
|
|
195
176
|
method: 'POST',
|
|
196
177
|
body: JSON.stringify({ language: 'pt-br', name: namePT }),
|
|
197
178
|
headers: { 'Content-Type': 'application/json' },
|
|
198
179
|
});
|
|
199
|
-
|
|
180
|
+
|
|
200
181
|
// โ
Store the generated basic info in KV
|
|
201
182
|
Object.assign(existingEN, {
|
|
202
183
|
name: nameEN,
|
|
@@ -205,7 +186,7 @@ export class ConceptService {
|
|
|
205
186
|
status: 'idle',
|
|
206
187
|
});
|
|
207
188
|
await kvStore.put(kvKeyEN, JSON.stringify(existingEN));
|
|
208
|
-
|
|
189
|
+
|
|
209
190
|
Object.assign(existingPT, {
|
|
210
191
|
name: namePT,
|
|
211
192
|
description: descriptionPT,
|
|
@@ -213,11 +194,16 @@ export class ConceptService {
|
|
|
213
194
|
status: 'idle',
|
|
214
195
|
});
|
|
215
196
|
await kvStore.put(kvKeyPT, JSON.stringify(existingPT));
|
|
216
|
-
|
|
217
|
-
console.log(
|
|
197
|
+
|
|
198
|
+
console.log(
|
|
199
|
+
`โ
Stored basic info for ${conceptSlug}, combination: ${combinationString}.`
|
|
200
|
+
);
|
|
218
201
|
return;
|
|
219
202
|
} catch (error) {
|
|
220
|
-
console.error(
|
|
203
|
+
console.error(
|
|
204
|
+
`โ Attempt ${attempts} failed at phase: ${phase}`,
|
|
205
|
+
(error as Error).message
|
|
206
|
+
);
|
|
221
207
|
}
|
|
222
208
|
}
|
|
223
209
|
}
|
|
@@ -805,4 +791,434 @@ export class ConceptService {
|
|
|
805
791
|
status: 'idle',
|
|
806
792
|
};
|
|
807
793
|
}
|
|
794
|
+
|
|
795
|
+
private async findDuplicateNamesAndUsedNames(limit?: number) {
|
|
796
|
+
console.log(`๐ Scanning KV for duplicate names...`);
|
|
797
|
+
|
|
798
|
+
const kvStore = this.context.env.KV_CONCEPTS;
|
|
799
|
+
let cursor: string | undefined = undefined;
|
|
800
|
+
let nameUsageMapEN: Record<string, string[]> = {}; // { enName: [combination1, combination2] }
|
|
801
|
+
let usedNamesEN = new Set<string>(); // โ
All used EN names
|
|
802
|
+
let usedNamesPT = new Set<string>(); // โ
All used PT names
|
|
803
|
+
let totalKeysScanned = 0;
|
|
804
|
+
|
|
805
|
+
do {
|
|
806
|
+
const response: KVNamespaceListResult<KVConcept> = (await kvStore.list({
|
|
807
|
+
prefix: 'concepts:',
|
|
808
|
+
cursor,
|
|
809
|
+
})) as KVNamespaceListResult<KVConcept, string>;
|
|
810
|
+
|
|
811
|
+
const {
|
|
812
|
+
keys,
|
|
813
|
+
cursor: newCursor,
|
|
814
|
+
}: { keys: KVNamespaceListKey<KVConcept>[]; cursor?: string } = response;
|
|
815
|
+
|
|
816
|
+
totalKeysScanned += keys.length;
|
|
817
|
+
|
|
818
|
+
for (const key of keys) {
|
|
819
|
+
const kvData = (await kvStore.get(key.name, 'json')) as {
|
|
820
|
+
name: string;
|
|
821
|
+
ptName?: string;
|
|
822
|
+
};
|
|
823
|
+
|
|
824
|
+
if (kvData?.name) {
|
|
825
|
+
usedNamesEN.add(kvData.name); // โ
Store all used EN names
|
|
826
|
+
|
|
827
|
+
if (!nameUsageMapEN[kvData.name]) {
|
|
828
|
+
nameUsageMapEN[kvData.name] = [];
|
|
829
|
+
}
|
|
830
|
+
nameUsageMapEN[kvData.name].push(key.name); // Store the key for that name
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
if (kvData?.ptName) {
|
|
834
|
+
usedNamesPT.add(kvData.ptName); // โ
Store all used PT names
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
cursor = newCursor;
|
|
839
|
+
} while (cursor && (!limit || totalKeysScanned < limit));
|
|
840
|
+
|
|
841
|
+
// โ
Find names used more than once (EN)
|
|
842
|
+
let duplicateENNames = Object.entries(nameUsageMapEN)
|
|
843
|
+
.filter(([_, combinations]) => combinations.length > 1)
|
|
844
|
+
.map(([name, combinations]) => ({
|
|
845
|
+
name,
|
|
846
|
+
combinations,
|
|
847
|
+
}));
|
|
848
|
+
|
|
849
|
+
// โ
If limit is set, trim duplicates list
|
|
850
|
+
if (limit && duplicateENNames.length > limit) {
|
|
851
|
+
duplicateENNames = duplicateENNames.slice(0, limit);
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
console.log(
|
|
855
|
+
`โ
Found ${duplicateENNames.length} duplicate EN names out of ${totalKeysScanned} entries.`
|
|
856
|
+
);
|
|
857
|
+
|
|
858
|
+
return {
|
|
859
|
+
duplicateEntries: duplicateENNames, // โ
List of EN names that need regeneration
|
|
860
|
+
usedNamesEN, // โ
Set of all unique EN names
|
|
861
|
+
usedNamesPT, // โ
Set of all unique PT names
|
|
862
|
+
};
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
async regenerateDuplicateNames(limit?: number) {
|
|
866
|
+
console.log(`๐ Processing duplicate names for regeneration...`);
|
|
867
|
+
|
|
868
|
+
const { duplicateEntries, usedNamesEN, usedNamesPT } =
|
|
869
|
+
await this.findDuplicateNamesAndUsedNames(limit);
|
|
870
|
+
|
|
871
|
+
if (duplicateEntries.length === 0) {
|
|
872
|
+
console.log(`๐ No duplicates found. Nothing to regenerate.`);
|
|
873
|
+
return { message: 'No duplicates detected.' };
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
for (const { name: oldNameEN, combinations } of duplicateEntries) {
|
|
877
|
+
for (let i = 1; i < combinations.length; i++) {
|
|
878
|
+
const [_, lang, conceptSlug, combinationString] =
|
|
879
|
+
combinations[i].split(':');
|
|
880
|
+
|
|
881
|
+
if (!lang || !conceptSlug || !combinationString) {
|
|
882
|
+
console.warn(`โ ๏ธ Invalid KV key format: ${combinations[i]}`);
|
|
883
|
+
continue;
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
const kvKeyEN = buildConceptKVKey(
|
|
887
|
+
'en-us',
|
|
888
|
+
conceptSlug as Concept,
|
|
889
|
+
combinationString
|
|
890
|
+
);
|
|
891
|
+
const kvKeyPT = buildConceptKVKey(
|
|
892
|
+
'pt-br',
|
|
893
|
+
conceptSlug as Concept,
|
|
894
|
+
combinationString
|
|
895
|
+
);
|
|
896
|
+
|
|
897
|
+
const kvDataEN = (await this.context.env.KV_CONCEPTS.get(
|
|
898
|
+
kvKeyEN,
|
|
899
|
+
'json'
|
|
900
|
+
)) as {
|
|
901
|
+
name: string;
|
|
902
|
+
description: string;
|
|
903
|
+
};
|
|
904
|
+
const kvDataPT = (await this.context.env.KV_CONCEPTS.get(
|
|
905
|
+
kvKeyPT,
|
|
906
|
+
'json'
|
|
907
|
+
)) as {
|
|
908
|
+
name: string;
|
|
909
|
+
};
|
|
910
|
+
|
|
911
|
+
console.log(
|
|
912
|
+
`๐ญ Regenerating name for: ${kvKeyEN} (Old Name: "${oldNameEN}")`
|
|
913
|
+
);
|
|
914
|
+
|
|
915
|
+
let newNameEN: string | null = null;
|
|
916
|
+
let newNamePT: string | null = null;
|
|
917
|
+
let attempts = 0;
|
|
918
|
+
const maxAttempts = 3;
|
|
919
|
+
|
|
920
|
+
while (attempts < maxAttempts) {
|
|
921
|
+
attempts++;
|
|
922
|
+
|
|
923
|
+
const messages = this.context
|
|
924
|
+
.buildLLMMessages()
|
|
925
|
+
.regenerateConceptName({
|
|
926
|
+
oldNameEN: kvDataEN.name,
|
|
927
|
+
description: kvDataEN.description,
|
|
928
|
+
usedNamesEN: Array.from(usedNamesEN), // โ
Ensure array format
|
|
929
|
+
});
|
|
930
|
+
|
|
931
|
+
const aiResponse = await this.context
|
|
932
|
+
.api()
|
|
933
|
+
.callTogether.single(messages, {});
|
|
934
|
+
if (!aiResponse) {
|
|
935
|
+
console.warn(
|
|
936
|
+
`โ ๏ธ AI failed to generate a new name for ${kvKeyEN}, skipping.`
|
|
937
|
+
);
|
|
938
|
+
break;
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
// โ
Parse AI response to extract both EN and PT names
|
|
942
|
+
const [generatedEN, generatedPT] = aiResponse
|
|
943
|
+
.split('\n')
|
|
944
|
+
.map((s) => s.trim());
|
|
945
|
+
|
|
946
|
+
// โ
Validate that the generated names are not duplicates
|
|
947
|
+
if (!usedNamesEN.has(generatedEN) && !usedNamesPT.has(generatedPT)) {
|
|
948
|
+
newNameEN = generatedEN;
|
|
949
|
+
newNamePT = generatedPT;
|
|
950
|
+
break; // โ
Found unique names, exit retry loop
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
console.warn(
|
|
954
|
+
`โ ๏ธ AI suggested a duplicate name: EN - "${generatedEN}", PT - "${generatedPT}". Retrying... (Attempt ${attempts}/${maxAttempts})`
|
|
955
|
+
);
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
if (!newNameEN || !newNamePT) {
|
|
959
|
+
console.error(
|
|
960
|
+
`๐จ Failed to generate a unique name for ${kvKeyEN} after ${maxAttempts} attempts.`
|
|
961
|
+
);
|
|
962
|
+
continue;
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
console.log(
|
|
966
|
+
`โ
Storing new names for ${kvKeyEN}: EN - "${newNameEN}", PT - "${newNamePT}"`
|
|
967
|
+
);
|
|
968
|
+
|
|
969
|
+
// โ
Update KV with new names (EN)
|
|
970
|
+
Object.assign(kvDataEN, { name: newNameEN });
|
|
971
|
+
await this.context.env.KV_CONCEPTS.put(
|
|
972
|
+
kvKeyEN,
|
|
973
|
+
JSON.stringify(kvDataEN)
|
|
974
|
+
);
|
|
975
|
+
|
|
976
|
+
// โ
Update KV with new names (PT)
|
|
977
|
+
Object.assign(kvDataPT, { name: newNamePT });
|
|
978
|
+
await this.context.env.KV_CONCEPTS.put(
|
|
979
|
+
kvKeyPT,
|
|
980
|
+
JSON.stringify(kvDataPT)
|
|
981
|
+
);
|
|
982
|
+
|
|
983
|
+
// โ
Add new names to the used names sets to prevent future reuse
|
|
984
|
+
usedNamesEN.add(newNameEN);
|
|
985
|
+
usedNamesPT.add(newNamePT);
|
|
986
|
+
|
|
987
|
+
// โ
Stop if limit is reached
|
|
988
|
+
if (limit && --limit <= 0) {
|
|
989
|
+
console.log(`๐ฏ Limit reached, stopping further processing.`);
|
|
990
|
+
return { message: 'Regeneration limit reached.' };
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
console.log(`๐ Duplicate names regenerated successfully.`);
|
|
996
|
+
return { message: 'Duplicate names processed and regenerated.' };
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
private async findDuplicateNamesAndUsedNamesForConcept(
|
|
1000
|
+
conceptSlug: Concept,
|
|
1001
|
+
limit?: number
|
|
1002
|
+
) {
|
|
1003
|
+
console.log(
|
|
1004
|
+
`๐ Scanning KV for duplicate names in concept: ${conceptSlug}...`
|
|
1005
|
+
);
|
|
1006
|
+
|
|
1007
|
+
const kvStore = this.context.env.KV_CONCEPTS;
|
|
1008
|
+
let cursor: string | undefined = undefined;
|
|
1009
|
+
let nameUsageMapEN: Record<string, string[]> = {}; // { enName: [combination1, combination2] }
|
|
1010
|
+
let usedNamesEN = new Set<string>(); // โ
All used EN names
|
|
1011
|
+
let usedNamesPT = new Set<string>(); // โ
All used PT names
|
|
1012
|
+
let totalKeysScanned = 0;
|
|
1013
|
+
|
|
1014
|
+
do {
|
|
1015
|
+
const response: KVNamespaceListResult<KVConcept> = (await kvStore.list({
|
|
1016
|
+
prefix: `concepts:en-us:${conceptSlug}:`, // โ
Fetch only keys for this concept
|
|
1017
|
+
cursor,
|
|
1018
|
+
})) as KVNamespaceListResult<KVConcept, string>;
|
|
1019
|
+
|
|
1020
|
+
const {
|
|
1021
|
+
keys,
|
|
1022
|
+
cursor: newCursor,
|
|
1023
|
+
}: { keys: KVNamespaceListKey<KVConcept>[]; cursor?: string } = response;
|
|
1024
|
+
|
|
1025
|
+
totalKeysScanned += keys.length;
|
|
1026
|
+
|
|
1027
|
+
for (const key of keys) {
|
|
1028
|
+
const kvDataEN = (await kvStore.get(key.name, 'json')) as {
|
|
1029
|
+
name: string;
|
|
1030
|
+
};
|
|
1031
|
+
const kvKeyPT = key.name.replace(':en-us:', ':pt-br:'); // โ
Get corresponding PT key
|
|
1032
|
+
const kvDataPT = (await kvStore.get(kvKeyPT, 'json')) as {
|
|
1033
|
+
name: string;
|
|
1034
|
+
};
|
|
1035
|
+
|
|
1036
|
+
if (kvDataEN?.name) {
|
|
1037
|
+
usedNamesEN.add(kvDataEN.name); // โ
Store all used EN names
|
|
1038
|
+
|
|
1039
|
+
if (!nameUsageMapEN[kvDataEN.name]) {
|
|
1040
|
+
nameUsageMapEN[kvDataEN.name] = [];
|
|
1041
|
+
}
|
|
1042
|
+
nameUsageMapEN[kvDataEN.name].push(key.name); // Store the key for that name
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
if (kvDataPT?.name) {
|
|
1046
|
+
usedNamesPT.add(kvDataPT.name); // โ
Store all used PT names
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
cursor = newCursor;
|
|
1051
|
+
} while (cursor && (!limit || totalKeysScanned < limit));
|
|
1052
|
+
|
|
1053
|
+
// โ
Find names used more than once (EN)
|
|
1054
|
+
let duplicateENNames = Object.entries(nameUsageMapEN)
|
|
1055
|
+
.filter(([_, combinations]) => combinations.length > 1)
|
|
1056
|
+
.map(([name, combinations]) => ({
|
|
1057
|
+
name,
|
|
1058
|
+
combinations,
|
|
1059
|
+
}));
|
|
1060
|
+
|
|
1061
|
+
// โ
If limit is set, trim duplicates list
|
|
1062
|
+
if (limit && duplicateENNames.length > limit) {
|
|
1063
|
+
duplicateENNames = duplicateENNames.slice(0, limit);
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
console.log(
|
|
1067
|
+
`โ
Found ${duplicateENNames.length} duplicate EN names for concept "${conceptSlug}" out of ${totalKeysScanned} entries.`
|
|
1068
|
+
);
|
|
1069
|
+
|
|
1070
|
+
return {
|
|
1071
|
+
duplicateEntries: duplicateENNames, // โ
List of EN names that need regeneration
|
|
1072
|
+
usedNamesEN, // โ
Set of all unique EN names
|
|
1073
|
+
usedNamesPT, // โ
Set of all unique PT names
|
|
1074
|
+
};
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
async regenerateDuplicateNamesForConcept(
|
|
1078
|
+
conceptSlug: Concept,
|
|
1079
|
+
limit?: number
|
|
1080
|
+
) {
|
|
1081
|
+
console.log(`๐ Processing duplicate names for concept: ${conceptSlug}...`);
|
|
1082
|
+
|
|
1083
|
+
const { duplicateEntries, usedNamesEN, usedNamesPT } =
|
|
1084
|
+
await this.findDuplicateNamesAndUsedNamesForConcept(conceptSlug, limit);
|
|
1085
|
+
|
|
1086
|
+
if (duplicateEntries.length === 0) {
|
|
1087
|
+
console.log(
|
|
1088
|
+
`๐ No duplicates found for concept: ${conceptSlug}. Nothing to regenerate.`
|
|
1089
|
+
);
|
|
1090
|
+
return { message: `No duplicates detected for ${conceptSlug}.` };
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
for (const { name: oldNameEN, combinations } of duplicateEntries) {
|
|
1094
|
+
for (let i = 1; i < combinations.length; i++) {
|
|
1095
|
+
const [_, lang, concept, combinationString] =
|
|
1096
|
+
combinations[i].split(':');
|
|
1097
|
+
|
|
1098
|
+
if (
|
|
1099
|
+
!lang ||
|
|
1100
|
+
!concept ||
|
|
1101
|
+
!combinationString ||
|
|
1102
|
+
concept !== conceptSlug
|
|
1103
|
+
) {
|
|
1104
|
+
console.warn(`โ ๏ธ Invalid KV key format: ${combinations[i]}`);
|
|
1105
|
+
continue;
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
const kvKeyEN = buildConceptKVKey(
|
|
1109
|
+
'en-us',
|
|
1110
|
+
concept as Concept,
|
|
1111
|
+
combinationString
|
|
1112
|
+
);
|
|
1113
|
+
const kvKeyPT = buildConceptKVKey(
|
|
1114
|
+
'pt-br',
|
|
1115
|
+
concept as Concept,
|
|
1116
|
+
combinationString
|
|
1117
|
+
);
|
|
1118
|
+
|
|
1119
|
+
const kvDataEN = (await this.context.env.KV_CONCEPTS.get(
|
|
1120
|
+
kvKeyEN,
|
|
1121
|
+
'json'
|
|
1122
|
+
)) as {
|
|
1123
|
+
name: string;
|
|
1124
|
+
description: string;
|
|
1125
|
+
};
|
|
1126
|
+
const kvDataPT = (await this.context.env.KV_CONCEPTS.get(
|
|
1127
|
+
kvKeyPT,
|
|
1128
|
+
'json'
|
|
1129
|
+
)) as {
|
|
1130
|
+
name: string;
|
|
1131
|
+
};
|
|
1132
|
+
|
|
1133
|
+
console.log(
|
|
1134
|
+
`๐ญ Regenerating name for: ${kvKeyEN} (Old Name: "${oldNameEN}")`
|
|
1135
|
+
);
|
|
1136
|
+
|
|
1137
|
+
let newNameEN: string | null = null;
|
|
1138
|
+
let newNamePT: string | null = null;
|
|
1139
|
+
let attempts = 0;
|
|
1140
|
+
const maxAttempts = 3;
|
|
1141
|
+
|
|
1142
|
+
while (attempts < maxAttempts) {
|
|
1143
|
+
attempts++;
|
|
1144
|
+
|
|
1145
|
+
const messages = this.context
|
|
1146
|
+
.buildLLMMessages()
|
|
1147
|
+
.regenerateConceptName({
|
|
1148
|
+
oldNameEN: kvDataEN.name,
|
|
1149
|
+
description: kvDataEN.description,
|
|
1150
|
+
usedNamesEN: Array.from(usedNamesEN), // โ
Ensure array format
|
|
1151
|
+
});
|
|
1152
|
+
|
|
1153
|
+
const aiResponse = await this.context
|
|
1154
|
+
.api()
|
|
1155
|
+
.callTogether.single(messages, {});
|
|
1156
|
+
if (!aiResponse) {
|
|
1157
|
+
console.warn(
|
|
1158
|
+
`โ ๏ธ AI failed to generate a new name for ${kvKeyEN}, skipping.`
|
|
1159
|
+
);
|
|
1160
|
+
break;
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
// โ
Parse AI response to extract both EN and PT names
|
|
1164
|
+
const [generatedEN, generatedPT] = aiResponse
|
|
1165
|
+
.split('\n')
|
|
1166
|
+
.map((s) => s.trim());
|
|
1167
|
+
|
|
1168
|
+
// โ
Validate that the generated names are not duplicates
|
|
1169
|
+
if (!usedNamesEN.has(generatedEN) && !usedNamesPT.has(generatedPT)) {
|
|
1170
|
+
newNameEN = generatedEN;
|
|
1171
|
+
newNamePT = generatedPT;
|
|
1172
|
+
break; // โ
Found unique names, exit retry loop
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
console.warn(
|
|
1176
|
+
`โ ๏ธ AI suggested a duplicate name: EN - "${generatedEN}", PT - "${generatedPT}". Retrying... (Attempt ${attempts}/${maxAttempts})`
|
|
1177
|
+
);
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
if (!newNameEN || !newNamePT) {
|
|
1181
|
+
console.error(
|
|
1182
|
+
`๐จ Failed to generate a unique name for ${kvKeyEN} after ${maxAttempts} attempts.`
|
|
1183
|
+
);
|
|
1184
|
+
continue;
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
console.log(
|
|
1188
|
+
`โ
Storing new names for ${kvKeyEN}: EN - "${newNameEN}", PT - "${newNamePT}"`
|
|
1189
|
+
);
|
|
1190
|
+
|
|
1191
|
+
// โ
Update KV with new names (EN)
|
|
1192
|
+
Object.assign(kvDataEN, { name: newNameEN });
|
|
1193
|
+
await this.context.env.KV_CONCEPTS.put(
|
|
1194
|
+
kvKeyEN,
|
|
1195
|
+
JSON.stringify(kvDataEN)
|
|
1196
|
+
);
|
|
1197
|
+
|
|
1198
|
+
// โ
Update KV with new names (PT)
|
|
1199
|
+
Object.assign(kvDataPT, { name: newNamePT });
|
|
1200
|
+
await this.context.env.KV_CONCEPTS.put(
|
|
1201
|
+
kvKeyPT,
|
|
1202
|
+
JSON.stringify(kvDataPT)
|
|
1203
|
+
);
|
|
1204
|
+
|
|
1205
|
+
// โ
Add new names to the used names sets to prevent future reuse
|
|
1206
|
+
usedNamesEN.add(newNameEN);
|
|
1207
|
+
usedNamesPT.add(newNamePT);
|
|
1208
|
+
|
|
1209
|
+
// โ
Stop if limit is reached
|
|
1210
|
+
if (limit && --limit <= 0) {
|
|
1211
|
+
console.log(`๐ฏ Limit reached, stopping further processing.`);
|
|
1212
|
+
return { message: `Regeneration limit reached for ${conceptSlug}.` };
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
console.log(
|
|
1218
|
+
`๐ Duplicate names regenerated successfully for concept: ${conceptSlug}.`
|
|
1219
|
+
);
|
|
1220
|
+
return {
|
|
1221
|
+
message: `Duplicate names processed and regenerated for ${conceptSlug}.`,
|
|
1222
|
+
};
|
|
1223
|
+
}
|
|
808
1224
|
}
|
package/package.json
CHANGED
package/utils/buildMessages.ts
CHANGED
|
@@ -104,7 +104,7 @@ export const buildLLMMessages = (env: BackendBindings) => ({
|
|
|
104
104
|
existingNames: string[];
|
|
105
105
|
}): ChatMessages => {
|
|
106
106
|
const zodiacCombination = mapConceptToPlanets(conceptSlug, combination);
|
|
107
|
-
|
|
107
|
+
|
|
108
108
|
return [
|
|
109
109
|
{
|
|
110
110
|
role: 'system',
|
|
@@ -179,4 +179,47 @@ export const buildLLMMessages = (env: BackendBindings) => ({
|
|
|
179
179
|
},
|
|
180
180
|
];
|
|
181
181
|
},
|
|
182
|
+
|
|
183
|
+
regenerateConceptName: ({
|
|
184
|
+
oldNameEN,
|
|
185
|
+
description,
|
|
186
|
+
usedNamesEN,
|
|
187
|
+
}: {
|
|
188
|
+
oldNameEN: string;
|
|
189
|
+
description: string;
|
|
190
|
+
usedNamesEN: string[];
|
|
191
|
+
}): ChatMessages => [
|
|
192
|
+
{
|
|
193
|
+
role: 'system',
|
|
194
|
+
content: `
|
|
195
|
+
You are an expert in creating mystical and evocative names for spiritual artifacts.
|
|
196
|
+
|
|
197
|
+
## Instructions:
|
|
198
|
+
- The name "${oldNameEN}" is already in use. Generate a new unique name that preserves the essence but avoids duplication.
|
|
199
|
+
- Keep it concise (โค5 words) and memorable.
|
|
200
|
+
- Ensure the name feels symbolic and mystical, fitting within the theme of arcane wisdom.
|
|
201
|
+
- Do NOT use astrological signs or zodiac references.
|
|
202
|
+
- Provide a natural Portuguese translation of the new name.
|
|
203
|
+
- Avoid these names: ${
|
|
204
|
+
usedNamesEN.length ? usedNamesEN.join(', ') : 'None'
|
|
205
|
+
}.
|
|
206
|
+
|
|
207
|
+
## Output Format:
|
|
208
|
+
EN: [new unique name]
|
|
209
|
+
PT: [Portuguese translation]
|
|
210
|
+
`,
|
|
211
|
+
},
|
|
212
|
+
{
|
|
213
|
+
role: 'user',
|
|
214
|
+
content: `
|
|
215
|
+
The current name "${oldNameEN}" has been found to be a duplicate.
|
|
216
|
+
|
|
217
|
+
### Description:
|
|
218
|
+
${description}
|
|
219
|
+
|
|
220
|
+
Generate a unique and evocative English name while maintaining the essence of the concept.
|
|
221
|
+
Also, provide a natural Portuguese translation of the new name.
|
|
222
|
+
`,
|
|
223
|
+
},
|
|
224
|
+
],
|
|
182
225
|
});
|