@zodic/shared 0.0.181 → 0.0.183

package/app/durable/ConceptNameDO.ts

@@ -1,4 +1,5 @@
 import { DurableObjectState } from '@cloudflare/workers-types';
+import { ConceptProgress } from '../../types';
 
 export class ConceptNameDO {
   state: DurableObjectState;
@@ -6,6 +7,7 @@ export class ConceptNameDO {
   names: { 'en-us': string[]; 'pt-br': string[] };
   count: number;
   timestamps: number[];
+  lastIndex: number; // ✅ Stores last processed index for batch processing
 
   static TOTAL_NAMES = 1728; // Maximum total names to be generated
 
@@ -15,6 +17,7 @@ export class ConceptNameDO {
     this.names = { 'en-us': [], 'pt-br': [] };
     this.count = 0;
     this.timestamps = [];
+    this.lastIndex = 0; // ✅ Initialize lastIndex
   }
 
   async fetch(request: Request): Promise<Response> {
@@ -33,6 +36,12 @@ export class ConceptNameDO {
       });
     }
 
+    if (method === 'GET' && url.pathname === '/last-index') {
+      return new Response(JSON.stringify(await this.getLastIndex()), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+
     if (method === 'POST' && url.pathname === '/add-name') {
       const { language, name } = (await request.json()) as {
         language: 'en-us' | 'pt-br';
@@ -47,6 +56,12 @@ export class ConceptNameDO {
       return new Response('OK', { status: 200 });
     }
 
+    if (method === 'POST' && url.pathname === '/set-progress') {
+      const { lastIndex } = (await request.json()) as { lastIndex: number };
+      await this.setLastIndex(lastIndex);
+      return new Response('OK', { status: 200 });
+    }
+
     return new Response('Not Found', { status: 404 });
   }
 
@@ -60,28 +75,14 @@ export class ConceptNameDO {
     return this.names;
   }
 
-  async getProgress(): Promise<{
-    count: number;
-    total: number;
-    progress: string;
-    ratePerMinute: number;
-    estimatedTimeMinutes: number | 'N/A';
-  }> {
-    if (this.count === 0) {
-      this.count = (await this.state.storage.get<number>('count')) || 0;
-    }
-    if (!Array.isArray(this.timestamps) || this.timestamps.length === 0) {
-      this.timestamps =
-        (await this.state.storage.get<number[]>('timestamps')) || [];
-    }
+  async getProgress(): Promise<ConceptProgress> {
+    await this.loadCount();
+    await this.loadTimestamps();
 
     const now = Date.now();
     const oneMinuteAgo = now - 60 * 1000;
 
-    const recentTimestamps = Array.isArray(this.timestamps)
-      ? this.timestamps.filter((ts) => ts >= oneMinuteAgo)
-      : []; // ✅ Ensure it's an array before calling filter
-
+    const recentTimestamps = this.timestamps.filter((ts) => ts >= oneMinuteAgo);
     const ratePerMinute = recentTimestamps.length;
 
     const remainingNames = ConceptNameDO.TOTAL_NAMES - this.count;
@@ -91,6 +92,7 @@ export class ConceptNameDO {
     return {
       count: this.count,
       total: ConceptNameDO.TOTAL_NAMES,
+      lastIndex: this.lastIndex,
       progress:
         ((this.count / ConceptNameDO.TOTAL_NAMES) * 100).toFixed(2) + '%',
       ratePerMinute,
@@ -98,20 +100,25 @@ export class ConceptNameDO {
     };
   }
 
+  async getLastIndex(): Promise<number> {
+    if (this.lastIndex === 0) {
+      this.lastIndex = (await this.state.storage.get<number>('lastIndex')) || 0;
+    }
+    return this.lastIndex;
+  }
+
   async addName(language: 'en-us' | 'pt-br', name: string): Promise<void> {
-    await this.getNames(); // Ensure latest name list is loaded
-    await this.loadCount(); // ✅ Load latest count before modifying
+    await this.getNames();
+    await this.loadCount();
 
     if (!this.names[language].includes(name)) {
       this.names[language].push(name);
-      this.count += 1; // ✅ Increment count correctly
-
+      this.count += 1;
       this.recordTimestamp();
 
-      // ✅ **Store all updated values in a single put() call**
       await this.state.storage.put({
         names: this.names,
-        count: this.count, // ✅ Store updated count properly
+        count: this.count,
         timestamps: this.timestamps,
       });
 
@@ -121,27 +128,23 @@ export class ConceptNameDO {
     }
   }
 
+  async setLastIndex(lastIndex: number): Promise<void> {
+    this.lastIndex = lastIndex;
+    await this.state.storage.put('lastIndex', this.lastIndex);
+  }
+
   async loadCount(): Promise<void> {
-    const storedCount = await this.state.storage.get<number>('count');
-    if (typeof storedCount === 'number') {
-      this.count = storedCount; // ✅ Ensure we always load the latest count
-    } else {
-      this.count = 0; // ✅ Ensure count is initialized properly
-    }
+    this.count = (await this.state.storage.get<number>('count')) || 0;
+  }
+
+  async loadTimestamps(): Promise<void> {
+    this.timestamps =
+      (await this.state.storage.get<number[]>('timestamps')) || [];
   }
 
   recordTimestamp(): void {
     const now = Date.now();
-
-    // ✅ Ensure timestamps is always an array before modifying
-    if (!Array.isArray(this.timestamps)) {
-      this.timestamps = [];
-    }
-
-    // ✅ Keep only the last 60 timestamps
-    if (this.timestamps.length >= 60) {
-      this.timestamps.splice(0, this.timestamps.length - 59);
-    }
+    this.timestamps = this.timestamps.filter((ts) => now - ts < 60 * 60 * 1000);
     this.timestamps.push(now);
   }
 }

package/app/services/ConceptService.ts

@@ -28,51 +28,51 @@ export class ConceptService {
     console.log(
       `🚀 Generating basic info for concept: ${conceptSlug}, combination: ${combinationString}, override: ${override}`
     );
-
+  
     const kvStore = this.context.kvConceptsStore();
     const kvFailuresStore = this.context.kvConceptFailuresStore();
     const kvKeyEN = buildConceptKVKey('en-us', conceptSlug, combinationString);
     const kvKeyPT = buildConceptKVKey('pt-br', conceptSlug, combinationString);
     const failureKey = `failures:basic-info:${conceptSlug}:${combinationString}`;
-
+  
     // ✅ Use Durable Object stub
     const id = this.context.env.CONCEPT_NAMES_DO.idFromName(conceptSlug);
     const stub = this.context.env.CONCEPT_NAMES_DO.get(id);
-
+  
     let attempts = 0;
     const maxAttempts = 3;
-
+  
     while (attempts < maxAttempts) {
       let phase = 'generation';
       try {
         attempts++;
-        console.log(`🔄 Attempt ${attempts} to generate basic info...`);
-
+        console.log(`🔄 Attempt ${attempts} to generate basic info for ${conceptSlug}...`);
+  
         // ✅ Fetch the latest name list from Durable Object
+        console.log(`📡 Fetching names from Durable Object for ${conceptSlug}...`);
         let allNamesEN: string[] = [];
         let allNamesPT: string[] = [];
-        const response = await stub.fetch(
-          `https://internal/names`
-        );
+        const response = await stub.fetch(`https://internal/names`);
+  
         if (response.ok) {
-          const data = (await response.json()) as {
-            'en-us': string[];
-            'pt-br': string[];
-          };
+          const data = (await response.json()) as { 'en-us': string[]; 'pt-br': string[] };
           allNamesEN = data['en-us'] || [];
           allNamesPT = data['pt-br'] || [];
+          console.log(`✅ Retrieved ${allNamesEN.length} EN names and ${allNamesPT.length} PT names.`);
         } else {
-          console.log(`!-- Error fetching names from DO for ${conceptSlug}`);
-          console.log('!-- Response:', await response.text());
+          console.log(`❌ Error fetching names from DO for ${conceptSlug}: ${await response.text()}`);
         }
-
+  
         // ✅ Fetch existing KV data to backfill Durable Object if necessary
+        console.log(`📡 Fetching existing KV data for ${conceptSlug}...`);
         const existingEN = await this.getKVConcept(kvKeyEN);
         const existingPT = await this.getKVConcept(kvKeyPT);
-
-        // 🔥 If names already exist in KV but not in DO, add them
+  
+        console.log(`📝 Existing EN name: ${existingEN.name || 'None'}, PT name: ${existingPT.name || 'None'}`);
+  
+        // 🔥 If names exist in KV but not in DO, add them
         if (existingEN.name && !allNamesEN.includes(existingEN.name)) {
-          console.log(`⚡ Backfilling existing name to DO: ${existingEN.name}`);
+          console.log(`⚡ Backfilling existing EN name to DO: ${existingEN.name}`);
           await stub.fetch(`https://internal/add-name`, {
             method: 'POST',
             body: JSON.stringify({ language: 'en-us', name: existingEN.name }),
@@ -80,23 +80,22 @@ export class ConceptService {
           });
         }
         if (existingPT.name && !allNamesPT.includes(existingPT.name)) {
-          console.log(`⚡ Backfilling existing name to DO: ${existingPT.name}`);
+          console.log(`⚡ Backfilling existing PT name to DO: ${existingPT.name}`);
           await stub.fetch(`https://internal/add-name`, {
             method: 'POST',
             body: JSON.stringify({ language: 'pt-br', name: existingPT.name }),
             headers: { 'Content-Type': 'application/json' },
           });
         }
-
+  
         // ✅ Skip generation if basic info already exists and override is false
         if (!override && existingEN.name && existingPT.name) {
-          console.log(
-            `⚡ Basic info already exists for ${conceptSlug}, skipping.`
-          );
+          console.log(`⚡ Basic info already exists for ${conceptSlug}, skipping.`);
           return;
         }
-
-        const recentNamesEN = allNamesEN.slice(-144);
+  
+        console.log(`✏️ Generating new name...`);
+        const recentNamesEN = allNamesEN.slice(-60);
         const messages = this.context
           .buildLLMMessages()
           .generateConceptBasicInfo({
@@ -104,43 +103,44 @@ export class ConceptService {
             conceptSlug,
             existingNames: recentNamesEN,
           });
-
-        let aiResponse = await this.context
-          .api()
-          .callTogether.single(messages, {});
+  
+        let aiResponse = await this.context.api().callTogether.single(messages, {});
         if (!aiResponse) throw new Error(`❌ AI returned an empty response`);
-
+  
         phase = 'cleaning';
+        console.log(`🧼 Cleaning AI response...`);
         aiResponse = this.cleanAIResponse(aiResponse);
-
+  
         phase = 'parsing';
+        console.log(`📜 Parsing AI response...`);
         let { nameEN, descriptionEN, poemEN, namePT, descriptionPT, poemPT } =
           this.parseBasicInfoResponse(aiResponse, conceptSlug);
-
+  
+        console.log(`🎭 Generated names: EN - "${nameEN}", PT - "${namePT}"`);
+  
         // ✅ Check uniqueness before storing
         if (allNamesEN.includes(nameEN) || allNamesPT.includes(namePT)) {
-          console.warn(
-            `⚠️ Duplicate Name Detected: "${nameEN}" or "${namePT}"`
-          );
+          console.warn(`⚠️ Duplicate Name Detected: "${nameEN}" or "${namePT}"`);
           if (attempts >= maxAttempts)
             throw new Error(`🚨 Could not generate a unique name`);
           console.log('🔁 Retrying due to duplicate name...');
           continue;
         }
-
+  
+        console.log(`📝 Storing names in Durable Object...`);
         // ✅ **Immediately update Durable Object with the new name**
         await stub.fetch(`https://internal/add-name`, {
           method: 'POST',
           body: JSON.stringify({ language: 'en-us', name: nameEN }),
           headers: { 'Content-Type': 'application/json' },
         });
-
+  
         await stub.fetch(`https://internal/add-name`, {
           method: 'POST',
           body: JSON.stringify({ language: 'pt-br', name: namePT }),
           headers: { 'Content-Type': 'application/json' },
         });
-
+  
         // ✅ Store the generated basic info in KV
         Object.assign(existingEN, {
           name: nameEN,
@@ -149,7 +149,7 @@ export class ConceptService {
           status: 'idle',
         });
         await kvStore.put(kvKeyEN, JSON.stringify(existingEN));
-
+  
         Object.assign(existingPT, {
           name: namePT,
           description: descriptionPT,
@@ -157,17 +157,12 @@ export class ConceptService {
           status: 'idle',
         });
         await kvStore.put(kvKeyPT, JSON.stringify(existingPT));
-
-        console.log(
-          `✅ Stored basic info for ${conceptSlug}, combination: ${combinationString}.`
-        );
+  
+        console.log(`✅ Stored basic info for ${conceptSlug}, combination: ${combinationString}.`);
         return;
       } catch (error) {
-        console.error(
-          `❌ Attempt ${attempts} failed at phase: ${phase}`,
-          (error as Error).message
-        );
-
+        console.error(`❌ Attempt ${attempts} failed at phase: ${phase}`, (error as Error).message);
+  
         // ✅ Store failure details in KV for debugging
         await kvFailuresStore.put(
           failureKey,
@@ -180,12 +175,10 @@ export class ConceptService {
             timestamp: new Date().toISOString(),
           })
         );
-
+  
         if (attempts >= maxAttempts) {
           console.error(`🚨 All ${maxAttempts} attempts failed.`);
-          throw new Error(
-            `Failed to generate basic info after ${maxAttempts} attempts.`
-          );
+          throw new Error(`Failed to generate basic info after ${maxAttempts} attempts.`);
         }
         console.log('🔁 Retrying...');
         await new Promise((resolve) => setTimeout(resolve, 500));

package/app/workflow/ConceptWorkflow.ts

@@ -1,12 +1,16 @@
 import { inject, injectable } from 'inversify';
 import pMap from 'p-map';
 import { zodiacSignCombinations } from '../../data/zodiacSignCombinations';
-import { Concept, ConceptPhase } from '../../types';
+import { Concept, ConceptPhase, ConceptProgress } from '../../types';
+import { AppContext } from '../base';
 import { ConceptService } from '../services/ConceptService';
 
 @injectable()
 export class ConceptWorkflow {
-  constructor(@inject(ConceptService) private conceptService: ConceptService) {}
+  constructor(
+    @inject(AppContext) private context: AppContext,
+    @inject(ConceptService) private conceptService: ConceptService
+  ) {}
 
   async processSingle(
     conceptSlug: Concept,
@@ -66,34 +70,79 @@ export class ConceptWorkflow {
     conceptSlug: Concept,
     combinations: string[],
     phase: ConceptPhase,
-    override?: boolean
+    override: boolean = false
   ) {
     console.log(
       `🚀 Processing batch for concept: ${conceptSlug}, Phase: ${phase}, Total Combinations: ${combinations.length}`
     );
 
-    const concurrency = 20; // Adjust based on response speed
+    const concurrency = 20;
     const failedCombinations: string[] = [];
 
+    // ✅ Get Durable Object stub for progress tracking
+    const id = this.context.env.CONCEPT_NAMES_DO.idFromName(conceptSlug);
+    const stub = this.context.env.CONCEPT_NAMES_DO.get(id);
+
+    // ✅ Try to load last checkpoint from DO
+    let lastIndex = 0;
+    try {
+      const progressRes = await stub.fetch(`https://internal/progress`);
+      if (progressRes.ok) {
+        const progressData = (await progressRes.json()) as ConceptProgress;
+        lastIndex = progressData.lastIndex || 0;
+        console.log(`🔄 Resuming from index ${lastIndex}`);
+      } else {
+        console.log(`📌 No previous checkpoint found, starting from scratch.`);
+      }
+    } catch (error) {
+      console.error(`❌ Error fetching progress from DO:`, error);
+    }
+
+    // ✅ Filter unprocessed combinations
+    const remainingCombinations = combinations.slice(lastIndex);
+    console.log(
+      `⏳ Processing ${remainingCombinations.length} remaining combinations...`
+    );
+
     await pMap(
-      combinations,
-      async (combination) => {
+      remainingCombinations,
+      async (combination, index) => {
+        const currentIndex = lastIndex + index;
         try {
+          console.log(
+            `🛠️ Processing combination ${currentIndex}/${combinations.length}: ${combination}`
+          );
           await this.processSingle(conceptSlug, combination, phase, override);
+
+          // ✅ Save progress every 10 iterations in DO
+          if (currentIndex % 10 === 0) {
+            await stub.fetch(`https://internal/set-progress`, {
+              method: 'POST',
+              body: JSON.stringify({ lastIndex: currentIndex }),
+              headers: { 'Content-Type': 'application/json' },
+            });
+            console.log(`✅ Checkpoint saved at index ${currentIndex}`);
+          }
         } catch (error) {
           console.error(`❌ Error processing ${combination}:`, error);
-          failedCombinations.push(combination); // Store failed ones
+          failedCombinations.push(combination);
         }
       },
       { concurrency }
     );
 
+    console.log(
+      `✅ Batch processing completed for ${conceptSlug}, Phase: ${phase}`
+    );
+
+    // ✅ Retry failed combinations after all batches
     if (failedCombinations.length > 0) {
       console.warn(`⚠️ Retrying failed combinations:`, failedCombinations);
-      await this.processBatch(conceptSlug, failedCombinations, phase, override); // Retry
+      await this.processBatch(conceptSlug, failedCombinations, phase, override);
     }
+
     console.log(
-      `✅ Batch processing completed for concept: ${conceptSlug}, Phase: ${phase}`
+      `🎉 All combinations successfully processed for ${conceptSlug}!`
     );
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zodic/shared",
-  "version": "0.0.181",
+  "version": "0.0.183",
   "module": "index.ts",
   "type": "module",
   "publishConfig": {

package/types/scopes/generic.ts

@@ -402,6 +402,7 @@ export type AstroKVData = {
 export type ConceptProgress = {
   count: number;
   total: number;
+  lastIndex: number;
   progress: string; // Percentage as string (e.g., "75.32%")
   ratePerMinute: number;
   estimatedTimeMinutes: number | 'N/A';