@zodic/shared 0.0.182 → 0.0.184

package/app/durable/ConceptNameDO.ts

@@ -6,6 +6,7 @@ export class ConceptNameDO {
   names: { 'en-us': string[]; 'pt-br': string[] };
   count: number;
   timestamps: number[];
+  lastIndex: number; // ✅ Tracks last processed index
 
   static TOTAL_NAMES = 1728; // Maximum total names to be generated
 
@@ -15,12 +16,15 @@ export class ConceptNameDO {
     this.names = { 'en-us': [], 'pt-br': [] };
     this.count = 0;
     this.timestamps = [];
+    this.lastIndex = 0; // ✅ Initialize lastIndex
   }
 
   async fetch(request: Request): Promise<Response> {
     const url = new URL(request.url);
     const method = request.method;
 
+    console.log(`📥 DO Received Request: ${method} ${url.pathname}`);
+
     if (method === 'GET' && url.pathname === '/names') {
       return new Response(JSON.stringify(await this.getNames()), {
         headers: { 'Content-Type': 'application/json' },
@@ -33,17 +37,30 @@ export class ConceptNameDO {
       });
     }
 
+    if (method === 'GET' && url.pathname === '/last-index') {
+      return new Response(JSON.stringify(await this.getLastIndex()), {
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+
     if (method === 'POST' && url.pathname === '/add-name') {
-      const { language, name } = (await request.json()) as {
+      const { language, name, index } = (await request.json()) as {
         language: 'en-us' | 'pt-br';
         name: string;
+        index: number;
       };
 
       if (!['en-us', 'pt-br'].includes(language)) {
         return new Response('Invalid language', { status: 400 });
       }
 
-      await this.addName(language, name);
+      await this.addName(language, name, index);
+      return new Response('OK', { status: 200 });
+    }
+
+    if (method === 'POST' && url.pathname === '/set-progress') {
+      const { lastIndex } = (await request.json()) as { lastIndex: number };
+      await this.setLastIndex(lastIndex);
       return new Response('OK', { status: 200 });
     }
 
@@ -63,25 +80,19 @@ export class ConceptNameDO {
   async getProgress(): Promise<{
     count: number;
     total: number;
+    lastIndex: number;
     progress: string;
     ratePerMinute: number;
     estimatedTimeMinutes: number | 'N/A';
   }> {
-    if (this.count === 0) {
-      this.count = (await this.state.storage.get<number>('count')) || 0;
-    }
-    if (!Array.isArray(this.timestamps) || this.timestamps.length === 0) {
-      this.timestamps =
-        (await this.state.storage.get<number[]>('timestamps')) || [];
-    }
+    await this.loadCount();
+    await this.loadTimestamps();
+    await this.loadLastIndex(); // ✅ Ensure latest lastIndex is fetched
 
     const now = Date.now();
     const oneMinuteAgo = now - 60 * 1000;
 
-    const recentTimestamps = Array.isArray(this.timestamps)
-      ? this.timestamps.filter((ts) => ts >= oneMinuteAgo)
-      : []; // ✅ Ensure it's an array before calling filter
-
+    const recentTimestamps = this.timestamps.filter((ts) => ts >= oneMinuteAgo);
     const ratePerMinute = recentTimestamps.length;
 
     const remainingNames = ConceptNameDO.TOTAL_NAMES - this.count;
@@ -91,6 +102,7 @@ export class ConceptNameDO {
     return {
       count: this.count,
       total: ConceptNameDO.TOTAL_NAMES,
+      lastIndex: this.lastIndex, // ✅ Include lastIndex in response
       progress:
         ((this.count / ConceptNameDO.TOTAL_NAMES) * 100).toFixed(2) + '%',
       ratePerMinute,
@@ -98,50 +110,72 @@ export class ConceptNameDO {
     };
   }
 
-  async addName(language: 'en-us' | 'pt-br', name: string): Promise<void> {
-    await this.getNames(); // Ensure latest name list is loaded
-    await this.loadCount(); // ✅ Load latest count before modifying
+  async getLastIndex(): Promise<number> {
+    await this.loadLastIndex(); // ✅ Fetch lastIndex from storage
+    return this.lastIndex;
+  }
+
+  async addName(
+    language: 'en-us' | 'pt-br',
+    name: string,
+    index: number
+  ): Promise<void> {
+    await this.getNames();
+    await this.loadCount();
+    await this.loadLastIndex(); // ✅ Ensure lastIndex is loaded before modifying
 
     if (!this.names[language].includes(name)) {
       this.names[language].push(name);
-      this.count += 1; // ✅ Increment count correctly
+      this.count += 1;
 
       this.recordTimestamp();
 
-      // ✅ **Store all updated values in a single put() call**
+      // 🔥 **Ensure lastIndex is updated**
+      if (index > this.lastIndex) {
+        this.lastIndex = index;
+      }
+
+      // 🔥 **Store all updated values in a single put() call**
       await this.state.storage.put({
         names: this.names,
-        count: this.count, // ✅ Store updated count properly
+        count: this.count,
         timestamps: this.timestamps,
+        lastIndex: this.lastIndex, // ✅ Store the last processed index
       });
 
       console.log(
-        `✅ Added name "${name}" to ${language}, count is now ${this.count}`
+        `✅ Added name "${name}" to ${language}, count: ${this.count}, lastIndex: ${this.lastIndex}`
       );
     }
   }
 
+  async setLastIndex(lastIndex: number): Promise<void> {
+    this.lastIndex = lastIndex;
+    await this.state.storage.put('lastIndex', this.lastIndex);
+    console.log(`📌 Updated lastIndex to ${this.lastIndex}`);
+  }
+
   async loadCount(): Promise<void> {
-    const storedCount = await this.state.storage.get<number>('count');
-    if (typeof storedCount === 'number') {
-      this.count = storedCount; // ✅ Ensure we always load the latest count
+    this.count = (await this.state.storage.get<number>('count')) || 0;
+  }
+
+  async loadLastIndex(): Promise<void> {
+    const storedIndex = await this.state.storage.get<number>('lastIndex');
+    if (typeof storedIndex === 'number') {
+      this.lastIndex = storedIndex;
     } else {
-      this.count = 0; // ✅ Ensure count is initialized properly
+      this.lastIndex = 0; // ✅ Ensure it starts from 0
     }
   }
 
+  async loadTimestamps(): Promise<void> {
+    this.timestamps =
+      (await this.state.storage.get<number[]>('timestamps')) || [];
+  }
+
   recordTimestamp(): void {
     const now = Date.now();
-
-    // ✅ Ensure timestamps is always an array before modifying
-    if (!Array.isArray(this.timestamps)) {
-      this.timestamps = [];
-    }
-
-    // ✅ Keep only the last 60 timestamps
-    if (this.timestamps.length >= 60) {
-      this.timestamps.splice(0, this.timestamps.length - 59);
-    }
+    this.timestamps = this.timestamps.filter((ts) => now - ts < 60 * 60 * 1000);
     this.timestamps.push(now);
   }
 }

package/app/workflow/ConceptWorkflow.ts

@@ -1,12 +1,16 @@
 import { inject, injectable } from 'inversify';
 import pMap from 'p-map';
 import { zodiacSignCombinations } from '../../data/zodiacSignCombinations';
-import { Concept, ConceptPhase } from '../../types';
+import { Concept, ConceptPhase, ConceptProgress } from '../../types';
+import { AppContext } from '../base';
 import { ConceptService } from '../services/ConceptService';
 
 @injectable()
 export class ConceptWorkflow {
-  constructor(@inject(ConceptService) private conceptService: ConceptService) {}
+  constructor(
+    @inject(AppContext) private context: AppContext,
+    @inject(ConceptService) private conceptService: ConceptService
+  ) {}
 
   async processSingle(
     conceptSlug: Concept,
@@ -66,34 +70,79 @@ export class ConceptWorkflow {
     conceptSlug: Concept,
     combinations: string[],
     phase: ConceptPhase,
-    override?: boolean
+    override: boolean = false
   ) {
     console.log(
       `🚀 Processing batch for concept: ${conceptSlug}, Phase: ${phase}, Total Combinations: ${combinations.length}`
     );
 
-    const concurrency = 20; // Adjust based on response speed
+    const concurrency = 5;
     const failedCombinations: string[] = [];
 
+    // ✅ Get Durable Object stub for progress tracking
+    const id = this.context.env.CONCEPT_NAMES_DO.idFromName(conceptSlug);
+    const stub = this.context.env.CONCEPT_NAMES_DO.get(id);
+
+    // ✅ Try to load last checkpoint from DO
+    let lastIndex = 0;
+    try {
+      const progressRes = await stub.fetch(`https://internal/progress`);
+      if (progressRes.ok) {
+        const progressData = (await progressRes.json()) as ConceptProgress;
+        lastIndex = progressData.lastIndex || 0;
+        console.log(`🔄 Resuming from index ${lastIndex}`);
+      } else {
+        console.log(`📌 No previous checkpoint found, starting from scratch.`);
+      }
+    } catch (error) {
+      console.error(`❌ Error fetching progress from DO:`, error);
+    }
+
+    // ✅ Filter unprocessed combinations
+    const remainingCombinations = combinations.slice(lastIndex);
+    console.log(
+      `⏳ Processing ${remainingCombinations.length} remaining combinations...`
+    );
+
     await pMap(
-      combinations,
-      async (combination) => {
+      remainingCombinations,
+      async (combination, index) => {
+        const currentIndex = lastIndex + index;
         try {
+          console.log(
+            `🛠️ Processing combination ${currentIndex}/${combinations.length}: ${combination}`
+          );
           await this.processSingle(conceptSlug, combination, phase, override);
+
+          // ✅ Save progress every 10 iterations in DO
+          if (currentIndex % 10 === 0) {
+            await stub.fetch(`https://internal/set-progress`, {
+              method: 'POST',
+              body: JSON.stringify({ lastIndex: currentIndex }),
+              headers: { 'Content-Type': 'application/json' },
+            });
+            console.log(`✅ Checkpoint saved at index ${currentIndex}`);
+          }
         } catch (error) {
           console.error(`❌ Error processing ${combination}:`, error);
-          failedCombinations.push(combination); // Store failed ones
+          failedCombinations.push(combination);
         }
       },
       { concurrency }
     );
 
+    console.log(
+      `✅ Batch processing completed for ${conceptSlug}, Phase: ${phase}`
+    );
+
+    // ✅ Retry failed combinations after all batches
     if (failedCombinations.length > 0) {
       console.warn(`⚠️ Retrying failed combinations:`, failedCombinations);
-      await this.processBatch(conceptSlug, failedCombinations, phase, override); // Retry
+      await this.processBatch(conceptSlug, failedCombinations, phase, override);
     }
+
     console.log(
-      `✅ Batch processing completed for concept: ${conceptSlug}, Phase: ${phase}`
+      `🎉 All combinations successfully processed for ${conceptSlug}!`
     );
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zodic/shared",
-  "version": "0.0.182",
+  "version": "0.0.184",
   "module": "index.ts",
   "type": "module",
   "publishConfig": {

package/types/scopes/generic.ts

@@ -402,6 +402,7 @@ export type AstroKVData = {
 export type ConceptProgress = {
   count: number;
   total: number;
+  lastIndex: number;
   progress: string; // Percentage as string (e.g., "75.32%")
   ratePerMinute: number;
   estimatedTimeMinutes: number | 'N/A';