@social-mail/social-mail-web-server 1.8.21 → 1.8.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@social-mail/social-mail-web-server",
-  "version": "1.8.21",
+  "version": "1.8.23",
   "description": "## Phase 1",
   "main": "index.js",
   "type": "module",

package/src/reports/model/entities/Visitor.ts

@@ -37,6 +37,9 @@ export default class Visitor {
     @Column({ dataType: "Char", length: 400, nullable: true})
     deviceVendor: string;
 
+    @Column({ dataType: "Char", length: 400, nullable: true})
+    referrer: string;
+
     @Column({ dataType: "Char", length: 400, nullable: true})
     cpu: string;
 

package/src/reports/workflows/activities/AddActivityWorkflow.ts

@@ -9,6 +9,7 @@ import Sql from "@entity-access/entity-access/dist/sql/Sql.js";
 
 export interface IActivityInfo {
     p: any;
+    r: any;
     u: any;
     q: any;
     x: any;
@@ -50,11 +51,14 @@ export default class AddActivityWorkflow extends Workflow<{
             }
         } = this.input;
 
+        let { activity: { r: referrer } } = this.input;
+
         await db.visitActivities.statements.insert({
             visitID,
             event,
             page,
             query,
+            referrer,
             x,
             y,
             url
@@ -62,6 +66,8 @@ export default class AddActivityWorkflow extends Workflow<{
 
         const visitor = await db.visitors.statements.select({}, { visitorID });
 
+        referrer = visitor.referrer;
+
         await db.visitors.where(visitor, (p) => (v) => v.visitorID === p.visitorID)
             .update(void 0, (p) => (v) => ({
                 firstVisitID: Sql.coll.min(v.visits.map((vv) => vv.visitID)),
@@ -142,5 +148,9 @@ export default class AddActivityWorkflow extends Workflow<{
         if (country) {
             await insertReport(startHour, `/country/${country}`, { country });
         }
+        if (referrer) {
+            referrer = referrer.replace(/^https?\:\/\//i, "");
+            await insertReport(startHour, `/source/${referrer}`, { referrer });
+        }
     }
 }

package/src/reports/workflows/visitor-added/VisitorAddedWorkflow.ts

@@ -4,7 +4,7 @@ import ReportDbContext from "../../model/ReportDbContext.js";
 import myUaParser from "my-ua-parser";
 import GeoIPService from "../../../server/services/geo-ip/GeoIPService.js";
 
-export default class VisitorAddedWorkflow extends Workflow<{ visitorID }> {
+export default class VisitorAddedWorkflow extends Workflow<{ visitorID, referrer }> {
 
     run() {
         return this.parseUserAgent();
@@ -15,7 +15,7 @@ export default class VisitorAddedWorkflow extends Workflow<{ visitorID }> {
         @Inject db?: ReportDbContext,
         @Inject geo?: GeoIPService
     ) {
-        const { visitorID } = this.input;
+        const { visitorID, referrer } = this.input;
         const v = await db.visitors.statements.select({}, { visitorID });
 
         const c = await geo.getBlock(v.ipAddress ?? "0.0.0.0");
@@ -35,6 +35,7 @@ export default class VisitorAddedWorkflow extends Workflow<{ visitorID }> {
 
         await db.visitors.statements.update({
             country,
+            referrer,
             app,
             appVersion,
             os,

package/src/reports/wwwroot/routes/social-mail/reports/api/track/[event]/get.tsx

@@ -35,7 +35,16 @@ export default class extends Page<IActivityInfo> {
 
         const time = DateTime.now;
 
-        const visitor = await this.getVisitor(this.request.cookies["vid"], siteID);
+        const {
+            p,
+            u,
+            x,
+            y,
+            r,
+            q
+        } = this.query;
+
+        const visitor = await this.getVisitor(this.request.cookies["vid"], siteID, r);
 
         const visitorID = visitor.visitorID;
 
@@ -67,13 +76,6 @@ export default class extends Page<IActivityInfo> {
             this.response.cookie("vvid", visitID, { secure: true, httpOnly: true, maxAge: visitMaxAge });
         }
 
-        const {
-            p,
-            u,
-            x,
-            y,
-            q
-        } = this.query;
 
         await this.wc.queue(AddActivityWorkflow, {
             siteID,
@@ -87,14 +89,15 @@ export default class extends Page<IActivityInfo> {
                 u,
                 x,
                 y,
-                q
+                q,
+                r
             }
         });
 
         return this.json({});
     }
 
-    async getVisitor(visitorID: any, siteID) {
+    async getVisitor(visitorID: any, siteID, referrer) {
 
         const { db } = this;
 
@@ -118,7 +121,7 @@ export default class extends Page<IActivityInfo> {
         });
 
         visitorID = `${visitor.visitorID}`;
-        await this.wc.queue(VisitorAddedWorkflow, { visitorID });
+        await this.wc.queue(VisitorAddedWorkflow, { visitorID, referrer });
 
         this.response.cookie("vid", visitorID, { secure: true, httpOnly: true, maxAge });
 

package/src/server/workflows/search/IndexAppFilesWorkflow.ts

@@ -23,11 +23,18 @@ export default class IndexAppFilesWorkflow extends Workflow {
     }
 
     async run() {
-        await this.index();
+        let start = -1;
+        for(;;) {
+            start = await this.index(start);
+            if (!BigInt(start)) {
+                break;
+            }
+        }
     }
 
     @UniqueActivity
     async index(
+        start,
         @Inject db?: SocialMailContext,
         @Inject tempFileService?: TempFileService
     ) {
@@ -35,16 +42,33 @@ export default class IndexAppFilesWorkflow extends Workflow {
         db.verifyFilters = false;
         db.raiseEvents = false;
 
-        const files = await db.appFiles
-            .where(void 0, (p) => (x) => x.fileContentID !== null && x.fileContent.textDocumentID === null)
+        let q = db.appFiles
+            .where(void 0, (p) => (x) => x.fileContentID !== null && x.fileContent.textDocumentID === null);
+
+        if (BigInt(start) > 0) {
+            q = q.where({ start }, (p) => (x) => x.appFileID < p.start);
+        }
+
+        const files = await q
             .orderByDescending(void 0, (p) => (x) => x.appFileID)
             .limit(100)
             .include((x) => x.fileContent)
             .toArray();
 
+        if (!files.length) {
+            return 0;
+        }
+
         for (const iterator of files) {
-            await this.saveTextContent(iterator, tempFileService, db);
+            start = iterator.appFileID;
+            try {
+                await this.saveTextContent(iterator, tempFileService, db);
+            } catch (error) {
+                console.error(error);
+            }
         }
+
+        return start;
     }
 
     private async extractText(file: LocalFile, tempFileService: TempFileService, db: SocialMailContext) {