@zodic/shared 0.0.182 → 0.0.183

package/app/durable/ConceptNameDO.ts

@@ -1,4 +1,5 @@
 import { DurableObjectState } from '@cloudflare/workers-types';
+import { ConceptProgress } from '../../types';
 
 export class ConceptNameDO {
   state: DurableObjectState;
@@ -6,6 +7,7 @@ export class ConceptNameDO {
   names: { 'en-us': string[]; 'pt-br': string[] };
   count: number;
   timestamps: number[];
+  lastIndex: number; // ✅ Stores last processed index for batch processing
 
   static TOTAL_NAMES = 1728; // Maximum total names to be generated
 
@@ -15,6 +17,7 @@ export class ConceptNameDO {
     this.names = { 'en-us': [], 'pt-br': [] };
     this.count = 0;
     this.timestamps = [];
+    this.lastIndex = 0; // ✅ Initialize lastIndex
   }
 
   async fetch(request: Request): Promise<Response> {
@@ -33,6 +36,12 @@ export class ConceptNameDO {
       });
     }
 
+    if (method === 'GET' && url.pathname === '/last-index') {
+      return new Response(JSON.stringify(await this.getLastIndex()), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+
     if (method === 'POST' && url.pathname === '/add-name') {
       const { language, name } = (await request.json()) as {
         language: 'en-us' | 'pt-br';
@@ -47,6 +56,12 @@ export class ConceptNameDO {
       return new Response('OK', { status: 200 });
     }
 
+    if (method === 'POST' && url.pathname === '/set-progress') {
+      const { lastIndex } = (await request.json()) as { lastIndex: number };
+      await this.setLastIndex(lastIndex);
+      return new Response('OK', { status: 200 });
+    }
+
     return new Response('Not Found', { status: 404 });
   }
 
@@ -60,28 +75,14 @@ export class ConceptNameDO {
     return this.names;
   }
 
-  async getProgress(): Promise<{
-    count: number;
-    total: number;
-    progress: string;
-    ratePerMinute: number;
-    estimatedTimeMinutes: number | 'N/A';
-  }> {
-    if (this.count === 0) {
-      this.count = (await this.state.storage.get<number>('count')) || 0;
-    }
-    if (!Array.isArray(this.timestamps) || this.timestamps.length === 0) {
-      this.timestamps =
-        (await this.state.storage.get<number[]>('timestamps')) || [];
-    }
+  async getProgress(): Promise<ConceptProgress> {
+    await this.loadCount();
+    await this.loadTimestamps();
 
     const now = Date.now();
     const oneMinuteAgo = now - 60 * 1000;
 
-    const recentTimestamps = Array.isArray(this.timestamps)
-      ? this.timestamps.filter((ts) => ts >= oneMinuteAgo)
-      : []; // ✅ Ensure it's an array before calling filter
-
+    const recentTimestamps = this.timestamps.filter((ts) => ts >= oneMinuteAgo);
     const ratePerMinute = recentTimestamps.length;
 
     const remainingNames = ConceptNameDO.TOTAL_NAMES - this.count;
@@ -91,6 +92,7 @@ export class ConceptNameDO {
     return {
       count: this.count,
       total: ConceptNameDO.TOTAL_NAMES,
+      lastIndex: this.lastIndex,
       progress:
         ((this.count / ConceptNameDO.TOTAL_NAMES) * 100).toFixed(2) + '%',
       ratePerMinute,
@@ -98,20 +100,25 @@ export class ConceptNameDO {
     };
   }
 
+  async getLastIndex(): Promise<number> {
+    if (this.lastIndex === 0) {
+      this.lastIndex = (await this.state.storage.get<number>('lastIndex')) || 0;
+    }
+    return this.lastIndex;
+  }
+
   async addName(language: 'en-us' | 'pt-br', name: string): Promise<void> {
-    await this.getNames(); // Ensure latest name list is loaded
-    await this.loadCount(); // ✅ Load latest count before modifying
+    await this.getNames();
+    await this.loadCount();
 
     if (!this.names[language].includes(name)) {
       this.names[language].push(name);
-      this.count += 1; // ✅ Increment count correctly
-
+      this.count += 1;
       this.recordTimestamp();
 
-      // ✅ **Store all updated values in a single put() call**
       await this.state.storage.put({
         names: this.names,
-        count: this.count, // ✅ Store updated count properly
+        count: this.count,
         timestamps: this.timestamps,
       });
 
@@ -121,27 +128,23 @@ export class ConceptNameDO {
     }
   }
 
+  async setLastIndex(lastIndex: number): Promise<void> {
+    this.lastIndex = lastIndex;
+    await this.state.storage.put('lastIndex', this.lastIndex);
+  }
+
   async loadCount(): Promise<void> {
-    const storedCount = await this.state.storage.get<number>('count');
-    if (typeof storedCount === 'number') {
-      this.count = storedCount; // ✅ Ensure we always load the latest count
-    } else {
-      this.count = 0; // ✅ Ensure count is initialized properly
-    }
+    this.count = (await this.state.storage.get<number>('count')) || 0;
+  }
+
+  async loadTimestamps(): Promise<void> {
+    this.timestamps =
+      (await this.state.storage.get<number[]>('timestamps')) || [];
   }
 
   recordTimestamp(): void {
     const now = Date.now();
-
-    // ✅ Ensure timestamps is always an array before modifying
-    if (!Array.isArray(this.timestamps)) {
-      this.timestamps = [];
-    }
-
-    // ✅ Keep only the last 60 timestamps
-    if (this.timestamps.length >= 60) {
-      this.timestamps.splice(0, this.timestamps.length - 59);
-    }
+    this.timestamps = this.timestamps.filter((ts) => now - ts < 60 * 60 * 1000);
     this.timestamps.push(now);
   }
 }

package/app/workflow/ConceptWorkflow.ts

@@ -1,12 +1,16 @@
 import { inject, injectable } from 'inversify';
 import pMap from 'p-map';
 import { zodiacSignCombinations } from '../../data/zodiacSignCombinations';
-import { Concept, ConceptPhase } from '../../types';
+import { Concept, ConceptPhase, ConceptProgress } from '../../types';
+import { AppContext } from '../base';
 import { ConceptService } from '../services/ConceptService';
 
 @injectable()
 export class ConceptWorkflow {
-  constructor(@inject(ConceptService) private conceptService: ConceptService) {}
+  constructor(
+    @inject(AppContext) private context: AppContext,
+    @inject(ConceptService) private conceptService: ConceptService
+  ) {}
 
   async processSingle(
     conceptSlug: Concept,
@@ -66,34 +70,79 @@ export class ConceptWorkflow {
     conceptSlug: Concept,
     combinations: string[],
     phase: ConceptPhase,
-    override?: boolean
+    override: boolean = false
   ) {
     console.log(
       `🚀 Processing batch for concept: ${conceptSlug}, Phase: ${phase}, Total Combinations: ${combinations.length}`
     );
 
-    const concurrency = 20; // Adjust based on response speed
+    const concurrency = 20;
     const failedCombinations: string[] = [];
 
+    // ✅ Get Durable Object stub for progress tracking
+    const id = this.context.env.CONCEPT_NAMES_DO.idFromName(conceptSlug);
+    const stub = this.context.env.CONCEPT_NAMES_DO.get(id);
+
+    // ✅ Try to load last checkpoint from DO
+    let lastIndex = 0;
+    try {
+      const progressRes = await stub.fetch(`https://internal/progress`);
+      if (progressRes.ok) {
+        const progressData = (await progressRes.json()) as ConceptProgress;
+        lastIndex = progressData.lastIndex || 0;
+        console.log(`🔄 Resuming from index ${lastIndex}`);
+      } else {
+        console.log(`📌 No previous checkpoint found, starting from scratch.`);
+      }
+    } catch (error) {
+      console.error(`❌ Error fetching progress from DO:`, error);
+    }
+
+    // ✅ Filter unprocessed combinations
+    const remainingCombinations = combinations.slice(lastIndex);
+    console.log(
+      `⏳ Processing ${remainingCombinations.length} remaining combinations...`
+    );
+
     await pMap(
-      combinations,
-      async (combination) => {
+      remainingCombinations,
+      async (combination, index) => {
+        const currentIndex = lastIndex + index;
         try {
+          console.log(
+            `🛠️ Processing combination ${currentIndex}/${combinations.length}: ${combination}`
+          );
           await this.processSingle(conceptSlug, combination, phase, override);
+
+          // ✅ Save progress every 10 iterations in DO
+          if (currentIndex % 10 === 0) {
+            await stub.fetch(`https://internal/set-progress`, {
+              method: 'POST',
+              body: JSON.stringify({ lastIndex: currentIndex }),
+              headers: { 'Content-Type': 'application/json' },
+            });
+            console.log(`✅ Checkpoint saved at index ${currentIndex}`);
+          }
         } catch (error) {
           console.error(`❌ Error processing ${combination}:`, error);
-          failedCombinations.push(combination); // Store failed ones
+          failedCombinations.push(combination);
         }
       },
       { concurrency }
     );
 
+    console.log(
+      `✅ Batch processing completed for ${conceptSlug}, Phase: ${phase}`
+    );
+
+    // ✅ Retry failed combinations after all batches
     if (failedCombinations.length > 0) {
       console.warn(`⚠️ Retrying failed combinations:`, failedCombinations);
-      await this.processBatch(conceptSlug, failedCombinations, phase, override); // Retry
+      await this.processBatch(conceptSlug, failedCombinations, phase, override);
     }
+
     console.log(
-      `✅ Batch processing completed for concept: ${conceptSlug}, Phase: ${phase}`
+      `🎉 All combinations successfully processed for ${conceptSlug}!`
     );
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zodic/shared",
-  "version": "0.0.182",
+  "version": "0.0.183",
   "module": "index.ts",
   "type": "module",
   "publishConfig": {

package/types/scopes/generic.ts

@@ -402,6 +402,7 @@ export type AstroKVData = {
 export type ConceptProgress = {
   count: number;
   total: number;
+  lastIndex: number;
   progress: string; // Percentage as string (e.g., "75.32%")
   ratePerMinute: number;
   estimatedTimeMinutes: number | 'N/A';