@social-mail/social-mail-web-server 1.8.49 → 1.8.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/workflows/search/IndexAppFilesWorkflow.d.ts.map +1 -1
- package/dist/server/workflows/search/IndexAppFilesWorkflow.js +27 -11
- package/dist/server/workflows/search/IndexAppFilesWorkflow.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/server/workflows/search/IndexAppFilesWorkflow.ts +28 -12
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IndexAppFilesWorkflow.d.ts","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"IndexAppFilesWorkflow.d.ts","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":"AACA,OAAO,QAA4B,MAAM,yDAAyD,CAAC;AAEnG,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAS/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAK9D,MAAM,CAAC,OAAO,OAAO,qBAAsB,SAAQ,QAAQ;IAEvD,MAAM,CAAC,SAAS,SAAW;IAE3B,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,kBAAkB;IAM1C,GAAG;IAWH,KAAK,CACP,KAAK,KAAA,EACG,EAAE,CAAC,EAAE,iBAAiB,EACtB,eAAe,CAAC,EAAE,eAAe;YAmC/B,WAAW;YAmCX,eAAe;CA0DhC"}
|
|
@@ -10,6 +10,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
10
10
|
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
11
11
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
12
12
|
};
|
|
13
|
+
/* eslint-disable no-console */
|
|
13
14
|
import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
|
|
14
15
|
import Inject, { ServiceProvider } from "@entity-access/entity-access/dist/di/di.js";
|
|
15
16
|
import SocialMailContext from "../../model/SocialMailContext.js";
|
|
@@ -20,6 +21,7 @@ import { parse } from "path";
|
|
|
20
21
|
import { Extract } from "../../services/extract/Extract.js";
|
|
21
22
|
import { appendFile } from "fs/promises";
|
|
22
23
|
import BufferHelper from "../../../common/BufferHelper.js";
|
|
24
|
+
import sleep from "../../../common/sleep.js";
|
|
23
25
|
class IndexAppFilesWorkflow extends Workflow {
|
|
24
26
|
static startIndexing(context) {
|
|
25
27
|
return context.queue(IndexAppFilesWorkflow, 0, {
|
|
@@ -92,6 +94,7 @@ class IndexAppFilesWorkflow extends Workflow {
|
|
|
92
94
|
return outputFile;
|
|
93
95
|
}
|
|
94
96
|
async saveTextContent(appFile, tempFileService, db) {
|
|
97
|
+
let searchable = "";
|
|
95
98
|
// check if file content already has text content to remove duplicates
|
|
96
99
|
const has = await db.textDocuments.statements.select({}, { textDocumentID: appFile.fileContentID });
|
|
97
100
|
if (has) {
|
|
@@ -101,18 +104,31 @@ class IndexAppFilesWorkflow extends Workflow {
|
|
|
101
104
|
textDocumentID: 0,
|
|
102
105
|
fileContentID: appFile.fileContentID
|
|
103
106
|
};
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
107
|
+
let file = null;
|
|
108
|
+
for (let i = 0; i < 3; i++) {
|
|
109
|
+
try {
|
|
110
|
+
file = await tempFileService.downloadAppFile(appFile);
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
// this should not happen...
|
|
115
|
+
console.error(error);
|
|
116
|
+
searchable = "";
|
|
117
|
+
await sleep(1000);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (file) {
|
|
121
|
+
const fx = await this.extractText(file, tempFileService, db);
|
|
122
|
+
if (fx.contentSize < 10 * 1024 * 1024) {
|
|
123
|
+
// we need to first remove 00 byte sequence...
|
|
124
|
+
const newFile = await tempFileService.createTempFile("txt");
|
|
125
|
+
const empty = Buffer.from("\n");
|
|
126
|
+
for await (const buffer of fx.readBuffers()) {
|
|
127
|
+
BufferHelper.replace00(buffer, empty);
|
|
128
|
+
await appendFile(newFile.path, buffer);
|
|
129
|
+
}
|
|
130
|
+
searchable = await newFile.readAsText();
|
|
114
131
|
}
|
|
115
|
-
searchable = await newFile.readAsText();
|
|
116
132
|
}
|
|
117
133
|
const td = await db.textDocuments.statements.upsert({
|
|
118
134
|
textDocumentID: changes.fileContentID,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IndexAppFilesWorkflow.js","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AACnG,OAAO,MAAM,EAAE,EAAE,eAAe,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAG/D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAGvD,OAAO,kBAAkB,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAE5D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,YAAY,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"IndexAppFilesWorkflow.js","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,+BAA+B;AAC/B,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AACnG,OAAO,MAAM,EAAE,EAAE,eAAe,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAG/D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAGvD,OAAO,kBAAkB,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAE5D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,YAAY,MAAM,iCAAiC,CAAC;AAC3D,OAAO,KAAK,MAAM,0BAA0B,CAAC;AAE7C,MAAqB,qBAAsB,SAAQ,QAAQ;IAIvD,MAAM,CAAC,aAAa,CAAC,OAA2B;QAC5C,OAAO,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE;YAC3C,EAAE,EAAE,iBAAiB;SACxB,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;QACf,SAAQ,CAAC;YACL,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAGK,AAAN,KAAK,CAAC,KAAK,CACP,KAAK,EACG,EAAsB,EACtB,eAAiC;QAGzC,EAAE,CAAC,aAAa,GAAG,KAAK,CAAC;QACzB,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC;QAEvB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ;aACd,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC;QAEpG,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC;aAChB,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;aACpD,KAAK,CAAC,GAAG,CAAC;aACV,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;aAC7B,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,CAAC;QACb,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC3B,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;YAC3B,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAC9D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAe,EAAE,eAAgC,EAAE,EAAqB;QAG9F,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAEhE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,KAAK,gBAAgB,CAAC;QACtD,IAAG,OAAO,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACnD,MAAM,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9C,KAAK,MAAM,QAAQ,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;gBAC7C,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,UAAU,CAAC,CAAC;gBACvD,MAAM,EAAE,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,GAAG,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9F,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;gBAC1D,IAAI,CAAC,EAAE,CAAC;oBACJ,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;YAED,OAAO,UAAU,CAAC;QACtB,CAAC;QAED,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;YAC3C,OAAO,UAAU,CAAC;QACtB,CAAC;QAAC,IAAI,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACrC,OAAO,UAAU,CAAC;QACtB,CAAC;QACD,MAAM,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,UAAU,CAAC;IACtB,CAAC;IAGO,KAAK,CAAC,eAAe,CAAC,OAAgB,EAAE,eAAgC,EAAE,EAAqB;QAEnG,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,sEAAsE;QACtE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;QACpG,IAAG,GAAG,EAAE,CAAC;YACL,OAAO;QACX,CAAC;QAED,MAAM,OAAO,GAAyB;YAClC,cAAc,EAAE,CAAC;YACjB,aAAa,EAAE,OAAO,CAAC,aAAa;SACvC,CAAC;QAEF,IAAI,IAAI,GAAc,IAAI,CAAC;QAE3B,KAAI,IAAI,CAAC,GAAC,CAAC,EAAC,CAAC,GAAC,CAAC,EAAC,CAAC,EAAE,EAAE,CAAC;YAClB,IAAI,CAAC;gBACD,IAAI,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;gBACtD,MAAM;YACV,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,4BAA4B;gBAC5B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACrB,UAAU,GAAG,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAE7D,IAAI,EAAE,CAAC,WAAW,GAAG,EAAE,GAAC,IAAI,GAAC,IAAI,EAAE,CAAC;gBAEhC,8CAA8C;gBAC9C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBAC5D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC1C,YAAY,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBACtC,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC3C,CAAC;gBAED,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAC5C,CAAC;QACL,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAChD,cAAc,EAAE,OAAO,CAAC,aAAa;YACrC,UAAU;SACb,CAAC,CAAC;QACH,OAAO,CAAC,cAAc,GAAG,EAAE,CAAC,cAAc,CAAC;QAE3C,MAAM,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CACnC,OAAO,CACV,CAAC;QAEF,OAAO;IACX,CAAC;;AArJM,+BAAS,GAAG,OAAO,CAAC;eAFV,qBAAqB;AAqBhC;IADL,cAAc;IAGV,WAAA,MAAM,CAAA;IACN,WAAA,MAAM,CAAA;;6CADM,iBAAiB;QACJ,eAAe;;kDAiC5C"}
|