@social-mail/social-mail-web-server 1.8.39 → 1.8.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/BufferHelper.d.ts +4 -0
- package/dist/common/BufferHelper.d.ts.map +1 -0
- package/dist/common/BufferHelper.js +15 -0
- package/dist/common/BufferHelper.js.map +1 -0
- package/dist/isolated/extract/IExtract.d.ts +3 -0
- package/dist/isolated/extract/IExtract.d.ts.map +1 -1
- package/dist/isolated/extract/index.d.ts +4 -2
- package/dist/isolated/extract/index.d.ts.map +1 -1
- package/dist/isolated/extract/index.js +9 -15
- package/dist/isolated/extract/index.js.map +1 -1
- package/dist/server/seed/ui/seed-ui.js +1 -1
- package/dist/server/services/convert/Convert.d.ts.map +1 -1
- package/dist/server/services/convert/Convert.js +6 -3
- package/dist/server/services/convert/Convert.js.map +1 -1
- package/dist/server/services/extract/Extract.d.ts +1 -1
- package/dist/server/services/extract/Extract.d.ts.map +1 -1
- package/dist/server/services/extract/Extract.js +24 -21
- package/dist/server/services/extract/Extract.js.map +1 -1
- package/dist/server/services/extract/pdf/PdfDoc.d.ts +1 -1
- package/dist/server/services/extract/pdf/PdfDoc.d.ts.map +1 -1
- package/dist/server/services/extract/pdf/PdfDoc.js +4 -4
- package/dist/server/services/extract/pdf/PdfDoc.js.map +1 -1
- package/dist/server/workflows/search/IndexAppFilesWorkflow.d.ts.map +1 -1
- package/dist/server/workflows/search/IndexAppFilesWorkflow.js +30 -39
- package/dist/server/workflows/search/IndexAppFilesWorkflow.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/common/BufferHelper.ts +18 -0
- package/src/isolated/extract/IExtract.ts +3 -0
- package/src/isolated/extract/index.ts +9 -15
- package/src/server/seed/ui/seed-ui.ts +1 -1
- package/src/server/services/convert/Convert.ts +9 -3
- package/src/server/services/extract/Extract.ts +24 -22
- package/src/server/services/extract/pdf/PdfDoc.ts +4 -4
- package/src/server/workflows/search/IndexAppFilesWorkflow.ts +39 -42
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BufferHelper.d.ts","sourceRoot":"","sources":["../../src/common/BufferHelper.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,YAAY;IAG7B,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAc9C"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export default class BufferHelper {
|
|
2
|
+
static replace00(b, replace) {
|
|
3
|
+
let start = 0;
|
|
4
|
+
for (;;) {
|
|
5
|
+
const i = b.indexOf(0x0, start);
|
|
6
|
+
if (i === -1) {
|
|
7
|
+
break;
|
|
8
|
+
}
|
|
9
|
+
b.set(replace, i);
|
|
10
|
+
start = i + 1;
|
|
11
|
+
}
|
|
12
|
+
return b;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=BufferHelper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BufferHelper.js","sourceRoot":"","sources":["../../src/common/BufferHelper.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,YAAY;IAG7B,MAAM,CAAC,SAAS,CAAC,CAAS,EAAE,OAAe;QACvC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,SAAQ,CAAC;YACL,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACX,MAAM;YACV,CAAC;YACD,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAClB,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,CAAC;IACb,CAAC;CAGJ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IExtract.d.ts","sourceRoot":"","sources":["../../../src/isolated/extract/IExtract.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,WAAW,QAAQ;IAC7B,KAAK,EAAE;QAAE,WAAW,MAAC;QAAC,QAAQ,MAAC;QAAC,IAAI,MAAA;KAAE,CAAC;IACvC,MAAM,EAAE,OAAO,CAAC;CACnB"}
|
|
1
|
+
{"version":3,"file":"IExtract.d.ts","sourceRoot":"","sources":["../../../src/isolated/extract/IExtract.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,WAAW,QAAQ;IAC7B,KAAK,EAAE;QAAE,WAAW,MAAC;QAAC,QAAQ,MAAC;QAAC,IAAI,MAAA;KAAE,CAAC;IACvC,MAAM,EAAE;QACJ,IAAI,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,MAAM,EAAE,OAAO,CAAC;CACnB"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import FileInfo from "../FileInfo.js";
|
|
2
2
|
import IExtract from "./IExtract.js";
|
|
3
3
|
export declare class ExtractWorker {
|
|
4
|
-
static text({ input: file, useOcr }: IExtract): Promise<
|
|
5
|
-
static ocr(file: FileInfo
|
|
4
|
+
static text({ input: file, useOcr, output }: IExtract): Promise<void>;
|
|
5
|
+
static ocr(file: FileInfo, output: {
|
|
6
|
+
path: string;
|
|
7
|
+
}): Promise<void>;
|
|
6
8
|
}
|
|
7
9
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/isolated/extract/index.ts"],"names":[],"mappings":"AAGA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,QAAQ,MAAM,eAAe,CAAC;AAGrC,qBAAa,aAAa;WACT,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/isolated/extract/index.ts"],"names":[],"mappings":"AAGA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,QAAQ,MAAM,eAAe,CAAC;AAGrC,qBAAa,aAAa;WACT,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ;WAe9C,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAC;CAK3D"}
|
|
@@ -1,32 +1,26 @@
|
|
|
1
1
|
/* eslint-disable no-console */
|
|
2
|
-
import { readFile } from "fs/promises";
|
|
2
|
+
import { copyFile, readFile, writeFile } from "fs/promises";
|
|
3
3
|
import tesseract from "tesseract.js";
|
|
4
4
|
const { recognize } = tesseract;
|
|
5
5
|
export class ExtractWorker {
|
|
6
|
-
static async text({ input: file, useOcr }) {
|
|
6
|
+
static async text({ input: file, useOcr, output }) {
|
|
7
7
|
try {
|
|
8
8
|
if (/^image\//.test(file.contentType)) {
|
|
9
9
|
if (useOcr) {
|
|
10
|
-
return this.ocr(file);
|
|
10
|
+
return this.ocr(file, output);
|
|
11
11
|
}
|
|
12
12
|
}
|
|
13
|
-
|
|
13
|
+
await copyFile(file.path, output.path);
|
|
14
|
+
return;
|
|
14
15
|
}
|
|
15
16
|
catch (error) {
|
|
16
17
|
console.error(error);
|
|
17
18
|
}
|
|
18
|
-
return "";
|
|
19
19
|
}
|
|
20
|
-
static async ocr(file) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
return result.data.text;
|
|
25
|
-
}
|
|
26
|
-
catch (error) {
|
|
27
|
-
console.error(error);
|
|
28
|
-
}
|
|
29
|
-
return "";
|
|
20
|
+
static async ocr(file, output) {
|
|
21
|
+
const buffer = await readFile(file.path);
|
|
22
|
+
const result = await recognize(buffer);
|
|
23
|
+
await writeFile(output.path, result.data.text, "utf-8");
|
|
30
24
|
}
|
|
31
25
|
}
|
|
32
26
|
;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/isolated/extract/index.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/isolated/extract/index.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,SAAS,MAAM,cAAc,CAAC;AAGrC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;AAEhC,MAAM,OAAO,aAAa;IACtB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAY;QACvD,IAAI,CAAC;YACD,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBACpC,IAAI,MAAM,EAAE,CAAC;oBACT,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;YAED,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACvC,OAAO;QACX,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAc,EAAE,MAAuB;QACpD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5D,CAAC;CACJ;AAAA,CAAC;AAGF,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;AAGzC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC5B,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;KAC3B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAG1B,yDAAyD;AACzD,YAAY;AACZ,iEAAiE;AACjE,8CAA8C;AAC9C,2BAA2B;AAC3B,wBAAwB;AACxB,gBAAgB;AAChB,kFAAkF;AAClF,0BAA0B;AAC1B,+EAA+E;AAC/E,YAAY;AACZ,2BAA2B;AAC3B,QAAQ;AACR,MAAM"}
|
|
@@ -3,7 +3,7 @@ export default async function seedUI(config) {
|
|
|
3
3
|
await config.saveVersion(UIPackageConfig, {
|
|
4
4
|
package: "@social-mail/social-mail-client",
|
|
5
5
|
view: "dist/web/AppIndex",
|
|
6
|
-
version: "1.8.
|
|
6
|
+
version: "1.8.216"
|
|
7
7
|
});
|
|
8
8
|
await config.saveVersion(WebComponentsPackageConfig, {
|
|
9
9
|
package: "@social-mail/web-components",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Convert.d.ts","sourceRoot":"","sources":["../../../../src/server/services/convert/Convert.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oDAAoD,CAAC;AAQ/E,eAAO,MAAM,OAAO;kBAEK,SAAS,UAAU,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,UAAU,MAAM;kBAyBlE,SAAS,QAAQ,MAAM,EAAE,OAAO,MAAM,UAAU,MAAM;
|
|
1
|
+
{"version":3,"file":"Convert.d.ts","sourceRoot":"","sources":["../../../../src/server/services/convert/Convert.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oDAAoD,CAAC;AAQ/E,eAAO,MAAM,OAAO;kBAEK,SAAS,UAAU,KAAK,GAAG,KAAK,GAAG,MAAM,GAAG,MAAM,UAAU,MAAM;kBAyBlE,SAAS,QAAQ,MAAM,EAAE,OAAO,MAAM,UAAU,MAAM;CA+C9E,CAAC"}
|
|
@@ -88,9 +88,12 @@ export const Convert = {
|
|
|
88
88
|
// txt hangs, so lets use to html and convert
|
|
89
89
|
args[1] = "html";
|
|
90
90
|
await this.execute(file, args, /\.html$/, output);
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
91
|
+
let text = "";
|
|
92
|
+
if (file.contentSize < 10 * 1024 * 1024) {
|
|
93
|
+
const htmlText = await readFile(output, "utf-8");
|
|
94
|
+
const { window } = new JSDOM(htmlText);
|
|
95
|
+
text = window.document.documentElement.innerText ?? window.document.body.innerText ?? htmlText;
|
|
96
|
+
}
|
|
94
97
|
await writeFile(output, text);
|
|
95
98
|
return output;
|
|
96
99
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Convert.js","sourceRoot":"","sources":["../../../../src/server/services/convert/Convert.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,MAAM,CAAC,MAAM,OAAO,GAAG;IAEnB,KAAK,CAAC,OAAO,CAAE,IAAe,EAAE,MAAuC,EAAE,MAAc;QACnF,IAAI,IAAc,CAAC;QACnB,IAAI,GAAG,GAAG,GAAG,CAAC;QACd,QAAO,MAAM,EAAE,CAAC;YACZ,KAAK,KAAK;gBACN,IAAI,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBAC/B,GAAG,GAAG,SAAS,CAAC;gBAChB,MAAM;YACV,KAAK,KAAK;gBACN,IAAI,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBAC/B,GAAG,GAAG,SAAS,CAAC;gBAChB,MAAM;YACV,KAAK,MAAM;gBACP,IAAI,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBAC/B,GAAG,GAAG,SAAS,CAAC;gBAChB,MAAM;YACV,KAAK,MAAM;gBACP,IAAI,GAAG,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;gBACjD,GAAG,GAAG,UAAU,CAAC;gBACjB,MAAM;QACd,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACjD,CAAC;IAGD,KAAK,CAAC,OAAO,CAAE,IAAe,EAAE,IAAc,EAAE,GAAW,EAAE,MAAc;QACvE,IAAI,CAAC;;;gBAED,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;oBACpB,6CAA6C;oBAC7C,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"Convert.js","sourceRoot":"","sources":["../../../../src/server/services/convert/Convert.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,MAAM,CAAC,MAAM,OAAO,GAAG;IAEnB,KAAK,CAAC,OAAO,CAAE,IAAe,EAAE,MAAuC,EAAE,MAAc;QACnF,IAAI,IAAc,CAAC;QACnB,IAAI,GAAG,GAAG,GAAG,CAAC;QACd,QAAO,MAAM,EAAE,CAAC;YACZ,KAAK,KAAK;gBACN,IAAI,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBAC/B,GAAG,GAAG,SAAS,CAAC;gBAChB,MAAM;YACV,KAAK,KAAK;gBACN,IAAI,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBAC/B,GAAG,GAAG,SAAS,CAAC;gBAChB,MAAM;YACV,KAAK,MAAM;gBACP,IAAI,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;gBAC/B,GAAG,GAAG,SAAS,CAAC;gBAChB,MAAM;YACV,KAAK,MAAM;gBACP,IAAI,GAAG,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;gBACjD,GAAG,GAAG,UAAU,CAAC;gBACjB,MAAM;QACd,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACjD,CAAC;IAGD,KAAK,CAAC,OAAO,CAAE,IAAe,EAAE,IAAc,EAAE,GAAW,EAAE,MAAc;QACvE,IAAI,CAAC;;;gBAED,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;oBACpB,6CAA6C;oBAC7C,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;oBAEjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;oBAElD,IAAI,IAAI,GAAG,EAAE,CAAC;oBACd,IAAI,IAAI,CAAC,WAAW,GAAG,EAAE,GAAC,IAAI,GAAC,IAAI,EAAE,CAAC;wBAElC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;wBACjD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;wBACvC,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAC;oBACnG,CAAC;oBACD,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;oBAC9B,OAAO,MAAM,CAAC;gBAClB,CAAC;gBAED,MAAM,GAAG,kCAAG,aAAa,CAAC,SAAS,CAAC,SAAS,CAAC,QAAA,CAAC;gBAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACzC,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACtD,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACxB,MAAM,YAAY,CAAC,aAAa,EAAE;oBAC9B,yBAAyB,MAAM,EAAE;oBACjC,YAAY;oBACZ,GAAI,IAAI;oBACR,UAAU,EAAE,GAAG,CAAC,MAAM;oBACtB,IAAI,CAAC,IAAI;oBACT,aAAa;oBACb,UAAU;iBACb,CAAC,CAAC;gBACH,KAAI,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;oBACpE,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;oBACvC,IAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;wBACtB,MAAM,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;wBAC1B,OAAO,MAAM,CAAC;oBAClB,CAAC;gBACL,CAAC;gBACD,OAAO,MAAM,CAAC;;;;;;;;;SACjB;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnD,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;CAEJ,CAAC;AAEF,oBAAoB;AACpB,mDAAmD;AACnD,yCAAyC;AACzC,yBAAyB;AACzB,SAAS;AACT,IAAI"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
|
|
2
2
|
export declare class Extract {
|
|
3
|
-
static text(src: LocalFile, useOcr?: boolean): Promise<
|
|
3
|
+
static text(src: LocalFile, outputFile: LocalFile, useOcr?: boolean): Promise<void>;
|
|
4
4
|
private static internalText;
|
|
5
5
|
}
|
|
6
6
|
//# sourceMappingURL=Extract.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Extract.d.ts","sourceRoot":"","sources":["../../../../src/server/services/extract/Extract.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oDAAoD,CAAC;
|
|
1
|
+
{"version":3,"file":"Extract.d.ts","sourceRoot":"","sources":["../../../../src/server/services/extract/Extract.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oDAAoD,CAAC;AAS/E,qBAAa,OAAO;WAEH,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,UAAQ;mBAIlD,YAAY;CAmDpC"}
|
|
@@ -51,22 +51,17 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
53
|
import { FileType } from "../../../common/FileType.js";
|
|
54
|
-
import {
|
|
54
|
+
import { copyFile } from "fs/promises";
|
|
55
55
|
import { Convert } from "../convert/Convert.js";
|
|
56
56
|
import LockFile from "../../storage/LockFile.js";
|
|
57
57
|
import PdfDoc from "./pdf/PdfDoc.js";
|
|
58
58
|
import IsolatedProcess from "../../../isolated/IsolatedProcess.js";
|
|
59
59
|
import { tempDiskCache } from "../../storage/tempDiskCache.js";
|
|
60
|
-
import { AppStringHelper } from "../../../common/AppStringHelper.js";
|
|
61
60
|
export class Extract {
|
|
62
|
-
static async text(src, useOcr = false) {
|
|
63
|
-
|
|
64
|
-
if (text) {
|
|
65
|
-
return AppStringHelper.remove00(text);
|
|
66
|
-
}
|
|
67
|
-
return text;
|
|
61
|
+
static async text(src, outputFile, useOcr = false) {
|
|
62
|
+
await this.internalText(src, outputFile, useOcr);
|
|
68
63
|
}
|
|
69
|
-
static async internalText(src, useOcr = false) {
|
|
64
|
+
static async internalText(src, outputFile, useOcr = false) {
|
|
70
65
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
71
66
|
try {
|
|
72
67
|
if (FileType.isMedia(src)) {
|
|
@@ -75,31 +70,39 @@ export class Extract {
|
|
|
75
70
|
// in future might provide some sort of
|
|
76
71
|
// caption extraction or title information
|
|
77
72
|
// extraction
|
|
78
|
-
|
|
73
|
+
await outputFile.writeAllText("");
|
|
74
|
+
return;
|
|
79
75
|
}
|
|
80
76
|
const folder = __addDisposableResource(env_1, tempDiskCache.newFolder("to-text-" + Date.now()), false);
|
|
81
77
|
const file = folder.get(src.fileName, src.contentType);
|
|
82
78
|
await src.copyTo(file);
|
|
83
79
|
if (FileType.isPdf(file)) {
|
|
84
|
-
|
|
80
|
+
await PdfDoc.extract(file, outputFile);
|
|
81
|
+
return;
|
|
85
82
|
}
|
|
86
83
|
if (FileType.isCode(file)) {
|
|
87
|
-
|
|
84
|
+
await copyFile(file.path, outputFile.path);
|
|
85
|
+
return;
|
|
88
86
|
}
|
|
89
87
|
if (!FileType.isImage(file)) {
|
|
90
88
|
if (/\.dat$/i.test(file.fileName)) {
|
|
91
89
|
return "";
|
|
92
90
|
}
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const { contentSize } = output;
|
|
96
|
-
if (contentSize > 10 * 1024 * 1024) {
|
|
97
|
-
throw new Error(`File size too big ${contentSize}`);
|
|
98
|
-
}
|
|
99
|
-
return await output.readAsText();
|
|
91
|
+
await Convert.convert(file, "text", outputFile.path);
|
|
92
|
+
return;
|
|
100
93
|
}
|
|
101
|
-
const
|
|
102
|
-
return await IsolatedProcess.extractText({
|
|
94
|
+
const _lock = __addDisposableResource(env_1, await LockFile.lock("extract-worker-lock"), false);
|
|
95
|
+
return await IsolatedProcess.extractText({
|
|
96
|
+
input: {
|
|
97
|
+
fileName: file.fileName,
|
|
98
|
+
contentType: file.contentType,
|
|
99
|
+
path: file.path
|
|
100
|
+
},
|
|
101
|
+
output: {
|
|
102
|
+
path: outputFile.path
|
|
103
|
+
},
|
|
104
|
+
useOcr
|
|
105
|
+
});
|
|
103
106
|
}
|
|
104
107
|
catch (e_1) {
|
|
105
108
|
env_1.error = e_1;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Extract.js","sourceRoot":"","sources":["../../../../src/server/services/extract/Extract.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,QAAQ,
|
|
1
|
+
{"version":3,"file":"Extract.js","sourceRoot":"","sources":["../../../../src/server/services/extract/Extract.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAY,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,QAAQ,MAAM,2BAA2B,CAAC;AACjD,OAAO,MAAM,MAAM,iBAAiB,CAAC;AACrC,OAAO,eAAe,MAAM,sCAAsC,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,MAAM,OAAO,OAAO;IAEhB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAc,EAAE,UAAqB,EAAE,MAAM,GAAG,KAAK;QACnE,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IACrD,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,GAAc,EAAE,UAAqB,EAAE,MAAM,GAAG,KAAK;;;YAEnF,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,wCAAwC;gBACxC,mCAAmC;gBAEnC,uCAAuC;gBACvC,0CAA0C;gBAC1C,aAAa;gBACb,MAAM,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;gBAClC,OAAO;YACX,CAAC;YAED,MAAM,MAAM,kCAAG,aAAa,CAAC,SAAS,CAAC,UAAU,GAAE,IAAI,CAAC,GAAG,EAAE,CAAC,QAAA,CAAC;YAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAEvB,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACvC,OAAO;YACX,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC3C,OAAO;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAE1B,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAChC,OAAO,EAAE,CAAC;gBACd,CAAC;gBAED,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;gBACrD,OAAO;YACX,CAAC;YAED,MAAM,KAAK,kCAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAA,CAAC;YAEzD,OAAO,MAAM,eAAe,CAAC,WAAW,CAAC;gBACrC,KAAK,EAAE;oBACH,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;iBAClB;gBACD,MAAM,EAAE;oBACJ,IAAI,EAAE,UAAU,CAAC,IAAI;iBACxB;gBACD,MAAM;aACT,CAAC,CAAC;;;;;;;;;KACN;CACJ;AAAA,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
|
|
2
2
|
export default class PdfDoc {
|
|
3
|
-
static extract(file: LocalFile): Promise<
|
|
3
|
+
static extract(file: LocalFile, outputFile: LocalFile): Promise<LocalFile>;
|
|
4
4
|
static extractAsHtmlToFile(src: LocalFile, output: LocalFile): Promise<void>;
|
|
5
5
|
}
|
|
6
6
|
//# sourceMappingURL=PdfDoc.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PdfDoc.d.ts","sourceRoot":"","sources":["../../../../../src/server/services/extract/pdf/PdfDoc.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oDAAoD,CAAC;AAM/E,MAAM,CAAC,OAAO,OAAO,MAAM;WAEV,OAAO,CAAC,IAAI,EAAE,SAAS;
|
|
1
|
+
{"version":3,"file":"PdfDoc.d.ts","sourceRoot":"","sources":["../../../../../src/server/services/extract/pdf/PdfDoc.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,oDAAoD,CAAC;AAM/E,MAAM,CAAC,OAAO,OAAO,MAAM;WAEV,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS;WAW9C,mBAAmB,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS;CAgCrE"}
|
|
@@ -56,15 +56,15 @@ import { join } from "node:path";
|
|
|
56
56
|
import DataUrl from "../../../../common/DataUrl.js";
|
|
57
57
|
import { tempDiskCache } from "../../../storage/tempDiskCache.js";
|
|
58
58
|
export default class PdfDoc {
|
|
59
|
-
static async extract(file) {
|
|
60
|
-
|
|
59
|
+
static async extract(file, outputFile) {
|
|
60
|
+
await spawnPromise("pdftotext", [
|
|
61
61
|
file.path,
|
|
62
|
-
|
|
62
|
+
outputFile.path,
|
|
63
63
|
], {
|
|
64
64
|
logData: false,
|
|
65
65
|
timeout: 5 * 60 * 1000
|
|
66
66
|
});
|
|
67
|
-
return
|
|
67
|
+
return outputFile;
|
|
68
68
|
}
|
|
69
69
|
static async extractAsHtmlToFile(src, output) {
|
|
70
70
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PdfDoc.js","sourceRoot":"","sources":["../../../../../src/server/services/extract/pdf/PdfDoc.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAElE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,OAAO,MAAM,+BAA+B,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAElE,MAAM,CAAC,OAAO,OAAO,MAAM;IAEvB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAe
|
|
1
|
+
{"version":3,"file":"PdfDoc.js","sourceRoot":"","sources":["../../../../../src/server/services/extract/pdf/PdfDoc.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAElE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,OAAO,MAAM,+BAA+B,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAElE,MAAM,CAAC,OAAO,OAAO,MAAM;IAEvB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAe,EAAE,UAAqB;QACvD,MAAM,YAAY,CAAC,WAAW,EAAE;YAC5B,IAAI,CAAC,IAAI;YACT,UAAU,CAAC,IAAI;SAClB,EAAE;YACC,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,CAAC,GAAC,EAAE,GAAC,IAAI;SACrB,CAAC,CAAC;QACH,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAc,EAAE,MAAiB;;;YAC9D,MAAM,MAAM,kCAAG,aAAa,CAAC,SAAS,CAAC,aAAa,CAAC,QAAA,CAAC;YACtD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACvB,MAAM,YAAY,CAAC,WAAW,EAAE;gBAC5B,IAAI;gBACJ,IAAI;gBACJ,IAAI,CAAC,IAAI;gBACT,OAAO;aACV,EAAE;gBACC,GAAG,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;YAE9D,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YAEpE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrD,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,GAAG;gBACnB,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,CAAC;aAC9E,CAAC,CAAC,CAAC,CAAC;YAEL,IAAI,IAAI,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;YAEzC,wBAAwB;YACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAClD,CAAC;YAED,MAAM,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;;;;;;;;;KACnC;CAEJ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IndexAppFilesWorkflow.d.ts","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":"AAAA,OAAO,QAA4B,MAAM,yDAAyD,CAAC;AAEnG,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAS/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"IndexAppFilesWorkflow.d.ts","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":"AAAA,OAAO,QAA4B,MAAM,yDAAyD,CAAC;AAEnG,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAS/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAI9D,MAAM,CAAC,OAAO,OAAO,qBAAsB,SAAQ,QAAQ;IAEvD,MAAM,CAAC,SAAS,SAAW;IAE3B,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,kBAAkB;IAM1C,GAAG;IAWH,KAAK,CACP,KAAK,KAAA,EACG,EAAE,CAAC,EAAE,iBAAiB,EACtB,eAAe,CAAC,EAAE,eAAe;YAmC/B,WAAW;YAmCX,eAAe;CAsChC"}
|
|
@@ -18,6 +18,8 @@ import { FileType } from "../../../common/FileType.js";
|
|
|
18
18
|
import EmailParserService from "../../smtp/services/EmailParserService.js";
|
|
19
19
|
import { parse } from "path";
|
|
20
20
|
import { Extract } from "../../services/extract/Extract.js";
|
|
21
|
+
import { appendFile } from "fs/promises";
|
|
22
|
+
import BufferHelper from "../../../common/BufferHelper.js";
|
|
21
23
|
class IndexAppFilesWorkflow extends Workflow {
|
|
22
24
|
static startIndexing(context) {
|
|
23
25
|
return context.queue(IndexAppFilesWorkflow, 0, {
|
|
@@ -61,70 +63,59 @@ class IndexAppFilesWorkflow extends Workflow {
|
|
|
61
63
|
return start;
|
|
62
64
|
}
|
|
63
65
|
async extractText(file, tempFileService, db) {
|
|
66
|
+
const outputFile = await tempFileService.createTempFile(".txt");
|
|
64
67
|
const isEmail = file.contentType === "message/rfc822";
|
|
65
68
|
if (isEmail) {
|
|
66
69
|
const parser = ServiceProvider.resolve(this, EmailParserService);
|
|
67
70
|
const parsedEmail = await parser.parse(file, true);
|
|
68
|
-
|
|
71
|
+
await outputFile.appendLine(parsedEmail.text);
|
|
69
72
|
for (const iterator of parsedEmail.attachments) {
|
|
70
73
|
const { ext } = parse(iterator.filename || "file.dat");
|
|
71
74
|
const tf = await tempFileService.createTempFile(ext, iterator.filename, iterator.contentType);
|
|
72
75
|
await tf.writeAll(iterator.content);
|
|
73
76
|
const t = await this.extractText(tf, tempFileService, db);
|
|
74
77
|
if (t) {
|
|
75
|
-
|
|
78
|
+
await outputFile.appendLine(await t.readAsText());
|
|
76
79
|
}
|
|
77
80
|
}
|
|
78
|
-
return
|
|
81
|
+
return outputFile;
|
|
79
82
|
}
|
|
80
83
|
if (/image\//i.test(file.contentType)) {
|
|
81
|
-
|
|
84
|
+
await Extract.text(file, outputFile, true);
|
|
85
|
+
return outputFile;
|
|
82
86
|
}
|
|
83
87
|
if (FileType.canBeCompressed(file)) {
|
|
84
|
-
|
|
88
|
+
await Extract.text(file, outputFile);
|
|
89
|
+
return outputFile;
|
|
85
90
|
}
|
|
86
|
-
|
|
91
|
+
await outputFile.writeAllText("");
|
|
92
|
+
return outputFile;
|
|
87
93
|
}
|
|
88
94
|
async saveTextContent(appFile, tempFileService, db) {
|
|
89
|
-
let searchable = "";
|
|
90
|
-
// const isEmail = appFile.contentType === "message/rfc822";
|
|
91
|
-
// if (isEmail) {
|
|
92
|
-
// ServiceProvider.resolve(this, MailPar)
|
|
93
|
-
// // this is email... it will contain the attachments..
|
|
94
|
-
// // const email = await db.emails.where(appFile, (p) => (x) => x.originalFileID === p.appFileID)
|
|
95
|
-
// // .include((x) => x.attachments)
|
|
96
|
-
// // .first();
|
|
97
|
-
// // if (email) {
|
|
98
|
-
// // const text = [email.textBody];
|
|
99
|
-
// // if (email.attachments) {
|
|
100
|
-
// // for (const attachment of email.attachments) {
|
|
101
|
-
// // text.push(attachment.name);
|
|
102
|
-
// // const t = await this.saveTextContent(attachment, tempFileService, db);
|
|
103
|
-
// // if (t) {
|
|
104
|
-
// // text.push(t);
|
|
105
|
-
// // }
|
|
106
|
-
// // }
|
|
107
|
-
// // }
|
|
108
|
-
// // searchable = text.join("\n");
|
|
109
|
-
// // }
|
|
110
|
-
// }
|
|
111
95
|
const changes = {
|
|
112
96
|
textDocumentID: 0,
|
|
113
97
|
fileContentID: appFile.fileContentID
|
|
114
98
|
};
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
99
|
+
const file = await tempFileService.downloadAppFile(appFile);
|
|
100
|
+
const fx = await this.extractText(file, tempFileService, db);
|
|
101
|
+
let searchable = "";
|
|
102
|
+
if (fx.contentSize < 10 * 1024 * 1024) {
|
|
103
|
+
// we need to first remove 00 byte sequence...
|
|
104
|
+
const newFile = await tempFileService.createTempFile("txt");
|
|
105
|
+
const empty = Buffer.from("\n");
|
|
106
|
+
for await (const buffer of fx.readBuffers()) {
|
|
107
|
+
BufferHelper.replace00(buffer, empty);
|
|
108
|
+
await appendFile(newFile.path, buffer);
|
|
109
|
+
}
|
|
110
|
+
searchable = await newFile.readAsText();
|
|
125
111
|
}
|
|
112
|
+
const td = await db.textDocuments.statements.upsert({
|
|
113
|
+
textDocumentID: changes.fileContentID,
|
|
114
|
+
searchable
|
|
115
|
+
});
|
|
116
|
+
changes.textDocumentID = td.textDocumentID;
|
|
126
117
|
await db.fileContents.statements.update(changes);
|
|
127
|
-
return
|
|
118
|
+
return;
|
|
128
119
|
}
|
|
129
120
|
}
|
|
130
121
|
IndexAppFilesWorkflow.taskGroup = "batch";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IndexAppFilesWorkflow.js","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AACnG,OAAO,MAAM,EAAE,EAAE,eAAe,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAG/D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAGvD,OAAO,kBAAkB,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"IndexAppFilesWorkflow.js","sourceRoot":"","sources":["../../../../src/server/workflows/search/IndexAppFilesWorkflow.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,OAAO,QAAQ,EAAE,EAAE,cAAc,EAAE,MAAM,yDAAyD,CAAC;AACnG,OAAO,MAAM,EAAE,EAAE,eAAe,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,iBAAiB,MAAM,kCAAkC,CAAC;AACjE,OAAO,eAAe,MAAM,kCAAkC,CAAC;AAG/D,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAGvD,OAAO,kBAAkB,MAAM,2CAA2C,CAAC;AAC3E,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,mCAAmC,CAAC;AAE5D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,YAAY,MAAM,iCAAiC,CAAC;AAE3D,MAAqB,qBAAsB,SAAQ,QAAQ;IAIvD,MAAM,CAAC,aAAa,CAAC,OAA2B;QAC5C,OAAO,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE;YAC3C,EAAE,EAAE,iBAAiB;SACxB,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;QACf,SAAQ,CAAC;YACL,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAGK,AAAN,KAAK,CAAC,KAAK,CACP,KAAK,EACG,EAAsB,EACtB,eAAiC;QAGzC,EAAE,CAAC,aAAa,GAAG,KAAK,CAAC;QACzB,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC;QAEvB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ;aACd,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,IAAI,IAAI,CAAC,CAAC,WAAW,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC;QAEpG,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC;aAChB,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;aACpD,KAAK,CAAC,GAAG,CAAC;aACV,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC;aAC7B,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,CAAC;QACb,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC3B,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;YAC3B,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;YAC9D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACzB,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAe,EAAE,eAAgC,EAAE,EAAqB;QAG9F,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAEhE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,KAAK,gBAAgB,CAAC;QACtD,IAAG,OAAO,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACnD,MAAM,UAAU,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9C,KAAK,MAAM,QAAQ,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;gBAC7C,MAAM,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,UAAU,CAAC,CAAC;gBACvD,MAAM,EAAE,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,GAAG,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9F,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACpC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;gBAC1D,IAAI,CAAC,EAAE,CAAC;oBACJ,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;YAED,OAAO,UAAU,CAAC;QACtB,CAAC;QAED,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;YAC3C,OAAO,UAAU,CAAC;QACtB,CAAC;QAAC,IAAI,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACrC,OAAO,UAAU,CAAC;QACtB,CAAC;QACD,MAAM,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,UAAU,CAAC;IACtB,CAAC;IAGO,KAAK,CAAC,eAAe,CAAC,OAAgB,EAAE,eAAgC,EAAE,EAAqB;QAEnG,MAAM,OAAO,GAAyB;YAClC,cAAc,EAAE,CAAC;YACjB,aAAa,EAAE,OAAO,CAAC,aAAa;SACvC,CAAC;QAEF,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE5D,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;QAE7D,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,IAAI,EAAE,CAAC,WAAW,GAAG,EAAE,GAAC,IAAI,GAAC,IAAI,EAAE,CAAC;YAEhC,8CAA8C;YAC9C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;gBAC1C,YAAY,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBACtC,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC3C,CAAC;YAED,UAAU,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC5C,CAAC;QAED,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAChD,cAAc,EAAE,OAAO,CAAC,aAAa;YACrC,UAAU;SACb,CAAC,CAAC;QACH,OAAO,CAAC,cAAc,GAAG,EAAE,CAAC,cAAc,CAAC;QAE3C,MAAM,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CACnC,OAAO,CACV,CAAC;QAEF,OAAO;IACX,CAAC;;AAjIM,+BAAS,GAAG,OAAO,CAAC;eAFV,qBAAqB;AAqBhC;IADL,cAAc;IAGV,WAAA,MAAM,CAAA;IACN,WAAA,MAAM,CAAA;;6CADM,iBAAiB;QACJ,eAAe;;kDAiC5C"}
|