@social-mail/social-mail-web-server 1.8.433 → 1.8.439

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/dist/server/model/SocialMailContext.d.ts +4 -0
  2. package/dist/server/model/SocialMailContext.d.ts.map +1 -1
  3. package/dist/server/model/SocialMailContext.js +4 -0
  4. package/dist/server/model/SocialMailContext.js.map +1 -1
  5. package/dist/server/model/SocialMailContextEvents.d.ts.map +1 -1
  6. package/dist/server/model/SocialMailContextEvents.js +6 -0
  7. package/dist/server/model/SocialMailContextEvents.js.map +1 -1
  8. package/dist/server/model/entities/Domain.d.ts.map +1 -1
  9. package/dist/server/model/entities/Domain.js +3 -2
  10. package/dist/server/model/entities/Domain.js.map +1 -1
  11. package/dist/server/model/entities/Email.d.ts +3 -0
  12. package/dist/server/model/entities/Email.d.ts.map +1 -1
  13. package/dist/server/model/entities/Email.js +9 -0
  14. package/dist/server/model/entities/Email.js.map +1 -1
  15. package/dist/server/model/entities/EmailAddress.d.ts +1 -0
  16. package/dist/server/model/entities/EmailAddress.d.ts.map +1 -1
  17. package/dist/server/model/entities/EmailAddress.js +12 -2
  18. package/dist/server/model/entities/EmailAddress.js.map +1 -1
  19. package/dist/server/model/entities/NameToken.d.ts +1 -1
  20. package/dist/server/model/entities/NameToken.d.ts.map +1 -1
  21. package/dist/server/model/entities/NameToken.js +5 -4
  22. package/dist/server/model/entities/NameToken.js.map +1 -1
  23. package/dist/server/model/entities/SearchToken.d.ts +8 -0
  24. package/dist/server/model/entities/SearchToken.d.ts.map +1 -0
  25. package/dist/server/model/entities/SearchToken.js +37 -0
  26. package/dist/server/model/entities/SearchToken.js.map +1 -0
  27. package/dist/server/model/entities/SearchTokenEmail.d.ts +12 -0
  28. package/dist/server/model/entities/SearchTokenEmail.d.ts.map +1 -0
  29. package/dist/server/model/entities/SearchTokenEmail.js +42 -0
  30. package/dist/server/model/entities/SearchTokenEmail.js.map +1 -0
  31. package/dist/server/model/entities/SearchWord.d.ts +1 -1
  32. package/dist/server/model/events/NameTokenEvents.d.ts +2 -4
  33. package/dist/server/model/events/NameTokenEvents.d.ts.map +1 -1
  34. package/dist/server/model/events/NameTokenEvents.js +2 -8
  35. package/dist/server/model/events/NameTokenEvents.js.map +1 -1
  36. package/dist/server/model/events/ReadOnlyEvents.d.ts +7 -0
  37. package/dist/server/model/events/ReadOnlyEvents.d.ts.map +1 -0
  38. package/dist/server/model/events/ReadOnlyEvents.js +16 -0
  39. package/dist/server/model/events/ReadOnlyEvents.js.map +1 -0
  40. package/dist/server/model/events/SearchTokenEmailEvents.d.ts +5 -0
  41. package/dist/server/model/events/SearchTokenEmailEvents.d.ts.map +1 -0
  42. package/dist/server/model/events/SearchTokenEmailEvents.js +4 -0
  43. package/dist/server/model/events/SearchTokenEmailEvents.js.map +1 -0
  44. package/dist/server/model/events/SearchTokenEvents.d.ts +5 -0
  45. package/dist/server/model/events/SearchTokenEvents.d.ts.map +1 -0
  46. package/dist/server/model/events/SearchTokenEvents.js +4 -0
  47. package/dist/server/model/events/SearchTokenEvents.js.map +1 -0
  48. package/dist/server/seed/ui/seed-ui.js +2 -2
  49. package/dist/server/services/EmailAddressService.d.ts.map +1 -1
  50. package/dist/server/services/EmailAddressService.js +2 -12
  51. package/dist/server/services/EmailAddressService.js.map +1 -1
  52. package/dist/server/services/emails/IndexEmailService.d.ts +14 -0
  53. package/dist/server/services/emails/IndexEmailService.d.ts.map +1 -0
  54. package/dist/server/services/emails/IndexEmailService.js +72 -0
  55. package/dist/server/services/emails/IndexEmailService.js.map +1 -0
  56. package/dist/server/services/extract/TextExtractorService.d.ts +9 -0
  57. package/dist/server/services/extract/TextExtractorService.d.ts.map +1 -0
  58. package/dist/server/services/extract/TextExtractorService.js +59 -0
  59. package/dist/server/services/extract/TextExtractorService.js.map +1 -0
  60. package/dist/server/services/search/SearchTokenService.d.ts +7 -0
  61. package/dist/server/services/search/SearchTokenService.d.ts.map +1 -0
  62. package/dist/server/services/search/SearchTokenService.js +107 -0
  63. package/dist/server/services/search/SearchTokenService.js.map +1 -0
  64. package/dist/server/workflows/SocialWorkflowContext.d.ts.map +1 -1
  65. package/dist/server/workflows/SocialWorkflowContext.js +4 -0
  66. package/dist/server/workflows/SocialWorkflowContext.js.map +1 -1
  67. package/dist/server/workflows/daily/DailyWorkflow.d.ts.map +1 -1
  68. package/dist/server/workflows/daily/DailyWorkflow.js +4 -0
  69. package/dist/server/workflows/daily/DailyWorkflow.js.map +1 -1
  70. package/dist/server/workflows/email/index/IndexEmailAddressWorkflow.d.ts +11 -0
  71. package/dist/server/workflows/email/index/IndexEmailAddressWorkflow.d.ts.map +1 -0
  72. package/dist/server/workflows/email/index/IndexEmailAddressWorkflow.js +69 -0
  73. package/dist/server/workflows/email/index/IndexEmailAddressWorkflow.js.map +1 -0
  74. package/dist/server/workflows/email/index/IndexEmailContentWorkflow.d.ts +11 -0
  75. package/dist/server/workflows/email/index/IndexEmailContentWorkflow.d.ts.map +1 -0
  76. package/dist/server/workflows/email/index/IndexEmailContentWorkflow.js +64 -0
  77. package/dist/server/workflows/email/index/IndexEmailContentWorkflow.js.map +1 -0
  78. package/dist/server/workflows/search/IndexFileContentsWorkflow.d.ts +2 -2
  79. package/dist/server/workflows/search/IndexFileContentsWorkflow.d.ts.map +1 -1
  80. package/dist/server/workflows/search/IndexFileContentsWorkflow.js +9 -39
  81. package/dist/server/workflows/search/IndexFileContentsWorkflow.js.map +1 -1
  82. package/dist/tsconfig.tsbuildinfo +1 -1
  83. package/package.json +2 -2
  84. package/src/server/model/SocialMailContext.ts +6 -0
  85. package/src/server/model/SocialMailContextEvents.ts +8 -0
  86. package/src/server/model/entities/Domain.ts +3 -2
  87. package/src/server/model/entities/Email.ts +10 -0
  88. package/src/server/model/entities/EmailAddress.ts +11 -2
  89. package/src/server/model/entities/NameToken.ts +5 -4
  90. package/src/server/model/entities/SearchToken.ts +26 -0
  91. package/src/server/model/entities/SearchTokenEmail.ts +34 -0
  92. package/src/server/model/entities/SearchWord.ts +1 -1
  93. package/src/server/model/events/NameTokenEvents.ts +2 -9
  94. package/src/server/model/events/ReadOnlyEvents.ts +20 -0
  95. package/src/server/model/events/SearchTokenEmailEvents.ts +6 -0
  96. package/src/server/model/events/SearchTokenEvents.ts +6 -0
  97. package/src/server/seed/ui/seed-ui.ts +2 -2
  98. package/src/server/services/EmailAddressService.ts +2 -13
  99. package/src/server/services/emails/IndexEmailService.ts +65 -0
  100. package/src/server/services/extract/TextExtractorService.ts +52 -0
  101. package/src/server/services/search/SearchTokenService.ts +33 -0
  102. package/src/server/workflows/SocialWorkflowContext.ts +5 -0
  103. package/src/server/workflows/daily/DailyWorkflow.ts +4 -0
  104. package/src/server/workflows/email/index/IndexEmailAddressWorkflow.ts +65 -0
  105. package/src/server/workflows/email/index/IndexEmailContentWorkflow.ts +54 -0
  106. package/src/server/workflows/search/IndexFileContentsWorkflow.ts +7 -43
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@social-mail/social-mail-web-server",
3
- "version": "1.8.433",
3
+ "version": "1.8.439",
4
4
  "description": "## Phase 1",
5
5
  "main": "index.js",
6
6
  "type": "module",
@@ -22,7 +22,7 @@
22
22
  "dependencies": {
23
23
  "@aws-sdk/client-s3": "^3.913.0",
24
24
  "@azure/storage-blob": "^12.29.1",
25
- "@entity-access/entity-access": "^1.0.526",
25
+ "@entity-access/entity-access": "^1.0.528",
26
26
  "@entity-access/server-pages": "^1.1.565",
27
27
  "@parse/node-apn": "^6.5.0",
28
28
  "@paypal/paypal-server-sdk": "^1.0.0",
@@ -80,6 +80,8 @@ import UserGroupMember from "./entities/UserGroupMember.js";
80
80
  import WebSiteTag from "./entities/WebSiteTag.js";
81
81
  import AppDbContext from "@entity-access/server-pages/dist/core/AppDbContext.js";
82
82
  import AppPassword from "./entities/AppPassword.js";
83
+ import SearchToken from "./entities/SearchToken.js";
84
+ import SearchTokenEmail from "./entities/SearchTokenEmail.js";
83
85
 
84
86
  @RegisterScoped
85
87
  export default class SocialMailContext extends AppDbContext {
@@ -178,6 +180,10 @@ export default class SocialMailContext extends AppDbContext {
178
180
 
179
181
  public subscriberActivities = this.model.register(SubscriberActivity);
180
182
 
183
+ public searchTokens = this.model.register(SearchToken);
184
+
185
+ public searchTokenEmails = this.model.register(SearchTokenEmail);
186
+
181
187
  public searchWords = this.model.register(SearchWord);
182
188
 
183
189
  public searchWordIndexes = this.model.register(SearchWordIndex);
@@ -146,6 +146,10 @@ import WebSiteTag from "./entities/WebSiteTag.js";
146
146
  import WebSiteTagEvents from "./events/WebSiteTagEvents.js";
147
147
  import AppPassword from "./entities/AppPassword.js";
148
148
  import AppPasswordEvents from "./events/AppPasswordEvents.js";
149
+ import SearchToken from "./entities/SearchToken.js";
150
+ import SearchTokenEvents from "./events/SearchTokenEvents.js";
151
+ import SearchTokenEmail from "./entities/SearchTokenEmail.js";
152
+ import SearchTokenEmailEvents from "./events/SearchTokenEmailEvents.js";
149
153
 
150
154
 
151
155
  @RegisterSingleton
@@ -195,6 +199,10 @@ export default class SocialMailContextEvents extends ContextEvents {
195
199
  this.register(MailboxFolder, MailboxFolderEvents);
196
200
  this.register(ChannelEmail, MailboxEmailEvents);
197
201
  this.register(NameToken, NameTokenEvents);
202
+
203
+ this.register(SearchToken, SearchTokenEvents);
204
+ this.register(SearchTokenEmail, SearchTokenEmailEvents);
205
+
198
206
  this.register(StoreItem, StoreItemEvents);
199
207
  this.register(StoreItemPrice, StoreItemPriceEvents);
200
208
  this.register(StoreAccount, StoreAccountEvents);
@@ -10,9 +10,10 @@ import type HostedZone from "./HostedZone.js";
10
10
 
11
11
  @Table("Domains")
12
12
  @Index({
13
- name: "IX_Unique_Domains",
13
+ name: "IX_Unique_Domains_V2",
14
+ dropNames:["IX_Unique_Domains"],
14
15
  unique: true,
15
- columns: [{ name: (x) => x.name, descending: false}]
16
+ columns: [{ name: (x) => x.name, descending: false, operatorClass: "varchar_pattern"}]
16
17
  })
17
18
  export default class Domain {
18
19
 
@@ -18,6 +18,7 @@ import Sql from "@entity-access/entity-access/dist/sql/Sql.js";
18
18
  import { apiPath } from "../../../common/globalEnv.js";
19
19
  import type MailboxContactEvent from "./MailboxContactEvent.js";
20
20
  import type ThreadParticipant from "./ThreadParticipant.js";
21
+ import type SearchTokenEmail from "./SearchTokenEmail.js";
21
22
 
22
23
  const emailStatuses = [ "queued", "received", "draft", "published", "pending-approval", "approved", "scheduled", "sent", "sent-failed", "deleted", "closed"] as const;
23
24
  export type emailStatusType = typeof emailStatuses[number];
@@ -73,6 +74,11 @@ export type emailStatusType = typeof emailStatuses[number];
73
74
  && x.status !== "draft"
74
75
  && x.status !== "scheduled"
75
76
  })
77
+ @Index({
78
+ name: "IX_Email_Pending_Search",
79
+ columns: [{ name: (x) => x.searchIndexed, descending: false }],
80
+ filter: (x) => x.searchIndexed === false && x.statePostReceiveInvoked === true
81
+ })
76
82
  export class Email {
77
83
 
78
84
  @Column({ key: true, dataType: "BigInt", generated: "identity" })
@@ -260,6 +266,9 @@ export class Email {
260
266
  @IgnoreJsonProperty
261
267
  statePostReceiveInvoked: boolean;
262
268
 
269
+ @Column( {dataType: "Boolean", default: () => false })
270
+ searchIndexed: boolean;
271
+
263
272
  @ReadOnlyJsonProperty
264
273
  postID: string;
265
274
 
@@ -314,6 +323,7 @@ export class Email {
314
323
  mailboxContactEvents: MailboxContactEvent[];
315
324
  threadParticipants: ThreadParticipant[];
316
325
  channelEmails: ChannelEmail[];
326
+ searchTokenEmails: SearchTokenEmail[];
317
327
  }
318
328
 
319
329
  @Table("EmailLogs")
@@ -16,10 +16,16 @@ import type MailboxContactEmail from "./MailboxContactEmail.js";
16
16
 
17
17
  @Table("EmailAddresses")
18
18
  @Index({
19
- name: "IX_Unique_EmailAddresses",
20
- columns: [{ name: (x) => x.emailAddress, descending : false }],
19
+ name: "IX_Unique_EmailAddresses_V2",
20
+ dropNames: [ "IX_Unique_EmailAddresses"],
21
+ columns: [{ name: (x) => x.emailAddress, descending : false, operatorClass: "varchar_pattern" }],
21
22
  unique: true
22
23
  })
24
+ @Index({
25
+ name: "IX_EmailAddresses_Pending_Name_Index",
26
+ columns: [ { name: (x) => x.emailAddressID, descending: false }],
27
+ filter: (x) => x.nameIndexed === false
28
+ })
23
29
  export default class EmailAddress {
24
30
 
25
31
  @Column({ key: true, generated: "identity", dataType: "BigInt"})
@@ -38,6 +44,9 @@ export default class EmailAddress {
38
44
  @Column({ dataType: "Char", length: 400 })
39
45
  public name: string;
40
46
 
47
+ @Column({ dataType: "Boolean", default: () => false})
48
+ public nameIndexed: boolean;
49
+
41
50
  @Column({ dataType: "Boolean", default: () => false})
42
51
  public blocked: boolean;
43
52
 
@@ -13,7 +13,7 @@ import type EmailAddressName from "./EmailAddressName.js";
13
13
  * Since one name will be linked with many addresses. It will require less
14
14
  * storage and faster access compared to inbuilt FTS.
15
15
  *
16
- * And This is the most importat part for Auto Complete.
16
+ * And This is the most important part for Auto Complete.
17
17
  *
18
18
  * Why store suggest column?
19
19
  * In dropdown we will only display suggestion if the name token has more than
@@ -26,15 +26,16 @@ import type EmailAddressName from "./EmailAddressName.js";
26
26
 
27
27
  @Table("NameTokens")
28
28
  @Index({
29
- name: "UX_NameTokens_Unique",
29
+ name: "UX_NameTokens_Unique_V2",
30
+ dropNames: [ "UX_NameTokens_Unique"],
30
31
  unique: true,
31
32
  columns: [
32
- { name: (x) => x.word, descending: false },
33
+ { name: (x) => x.word, descending: false, operatorClass: "varchar_pattern" },
33
34
  ]
34
35
  })
35
36
  @Index({
36
37
  name: "IX_NameTokens_Suggest",
37
- columns: [ { name: (x) => x.word, descending: false }],
38
+ columns: [ { name: (x) => x.word, descending: false, operatorClass: "varchar_pattern" }],
38
39
  filter: (x) => x.suggest === true
39
40
  })
40
41
  export default class NameToken {
@@ -0,0 +1,26 @@
1
+ import Column from "@entity-access/entity-access/dist/decorators/Column.js";
2
+ import Index from "@entity-access/entity-access/dist/decorators/Index.js";
3
+ import Table from "@entity-access/entity-access/dist/decorators/Table.js";
4
+ import type SearchTokenEmail from "./SearchTokenEmail.js";
5
+
6
+ @Table("SearchTokens")
7
+ @Index({
8
+ name: "UX_SearchTokens_Word",
9
+ columns: [ { name: (x) => x.word, descending: false, operatorClass: "varchar_pattern" } ],
10
+ include: [ (x) => x.doNotIndex],
11
+ unique: true
12
+ })
13
+ export default class SearchToken {
14
+
15
+ @Column({ dataType: "BigInt", key: true, generated: "identity"})
16
+ tokenID: number;
17
+
18
+ @Column({ dataType:"Char", length: 400, nullable: false })
19
+ word: string;
20
+
21
+ @Column({ dataType: "Boolean", default: () => false })
22
+ doNotIndex: boolean;
23
+
24
+ tokenEmails: SearchTokenEmail[];
25
+
26
+ }
@@ -0,0 +1,34 @@
1
+ import Column from "@entity-access/entity-access/dist/decorators/Column.js";
2
+ import { RelateTo } from "@entity-access/entity-access/dist/decorators/Relate.js";
3
+ import Table from "@entity-access/entity-access/dist/decorators/Table.js";
4
+ import SearchToken from "./SearchToken.js";
5
+ import { Email } from "./Email.js";
6
+
7
+ export const tokenKind = ["body", "attachment"] as const;
8
+ export type tokenKindType = typeof tokenKind[number];
9
+
10
+ @Table("SearchTokenEmails")
11
+ export default class SearchTokenEmail {
12
+
13
+ @Column({ dataType: "BigInt", key: true})
14
+ @RelateTo(SearchToken, {
15
+ property: (x) => x.token,
16
+ inverseProperty: (x) => x.tokenEmails,
17
+ })
18
+ tokenID: number;
19
+
20
+ @Column({ dataType: "Char", key: true, length: 20 })
21
+ tokenKind: tokenKindType;
22
+
23
+ @Column({ dataType: "BigInt", key: true })
24
+ @RelateTo(Email, {
25
+ property: (x) => x.email,
26
+ inverseProperty: (x) => x.searchTokenEmails
27
+ })
28
+ emailID: number;
29
+
30
+ token: SearchToken;
31
+
32
+ email: Email;
33
+
34
+ }
@@ -65,7 +65,7 @@ export class SearchWordIndex {
65
65
  fileContentID: number;
66
66
 
67
67
  /**
68
- * First two bytes represents the first word occurance (not character index) in the document.
68
+ * First two bytes represents the first word occurrence (not character index) in the document.
69
69
  * Last two bytes represents how many times it occurs in the document.
70
70
  * This is based on the fact that import words will appear in the title or the first paragraph
71
71
  * of the document.
@@ -1,15 +1,8 @@
1
- import { IEntityQuery } from "@entity-access/entity-access/dist/model/IFilterWithParameter.js";
2
1
  import NameToken from "../entities/NameToken.js";
3
- import AuthenticatedEvents from "./AuthenticatedEvents.js";
2
+ import ReadOnlyEvents from "./ReadOnlyEvents.js";
4
3
 
5
4
 
6
- export default class NameTokenEvents extends AuthenticatedEvents<NameToken> {
5
+ export default class NameTokenEvents extends ReadOnlyEvents<NameToken> {
7
6
 
8
- filter(query: IEntityQuery<NameToken>): IEntityQuery<NameToken> {
9
- if (this.verify) {
10
- this.sessionUser.ensureLoggedIn();
11
- }
12
- return query.limit(20);
13
- }
14
7
 
15
8
  }
@@ -0,0 +1,20 @@
1
+ import { IEntityQuery } from "@entity-access/entity-access/dist/model/IFilterWithParameter.js";
2
+ import AuthenticatedEvents from "./AuthenticatedEvents.js";
3
+
4
+ export default class ReadOnlyEvents<T> extends AuthenticatedEvents<T> {
5
+
6
+ filter(query: IEntityQuery<T>): IEntityQuery<T> {
7
+ if (this.verify) {
8
+ this.sessionUser.ensureLoggedIn();
9
+ }
10
+ return query.limit(20);
11
+ }
12
+
13
+ includeFilter(query: IEntityQuery<T>, type?: any, key?: string): IEntityQuery<T> {
14
+ if (this.verify) {
15
+ this.sessionUser.ensureLoggedIn();
16
+ }
17
+ return query;
18
+ }
19
+
20
+ }
@@ -0,0 +1,6 @@
1
+ import SearchTokenEmail from "../entities/SearchTokenEmail.js";
2
+ import ReadOnlyEvents from "./ReadOnlyEvents.js";
3
+
4
+ export default class SearchTokenEmailEvents extends ReadOnlyEvents<SearchTokenEmail> {
5
+
6
+ }
@@ -0,0 +1,6 @@
1
+ import SearchToken from "../entities/SearchToken.js";
2
+ import ReadOnlyEvents from "./ReadOnlyEvents.js";
3
+
4
+ export default class SearchTokenEvents extends ReadOnlyEvents<SearchToken> {
5
+
6
+ }
@@ -17,13 +17,13 @@ export default async function seedUI(config: DBConfig) {
17
17
  await config.saveVersion(UIPackageConfig, {
18
18
  package: "@social-mail/social-mail-client",
19
19
  view: "dist/web/AppIndex",
20
- version: "1.9.87"
20
+ version: "1.9.90"
21
21
  });
22
22
 
23
23
  await config.saveVersion(WebComponentsPackageConfig, {
24
24
  package: "@social-mail/web-components",
25
25
  view: "dist/index.js",
26
- version: "1.0.408"
26
+ version: "1.0.407"
27
27
  });
28
28
 
29
29
  await config.saveVersion(FAFreePackageConfig, {
@@ -5,8 +5,8 @@ import { globalServices } from "../../globalServices.js";
5
5
  import EntityAccessError from "@entity-access/entity-access/dist/common/EntityAccessError.js";
6
6
  import {ICacheContainer, ObjectCache, cacheFor } from "../../common/cacheFor.js";
7
7
  import Domain from "../model/entities/Domain.js";
8
- import RepairEmailNameTagsWorkflow from "../workflows/repair/tags/RepairEmailNameTagsWorkflow.js";
9
8
  import { AppWorkflowContext } from "../workflows/AppWorkflowContext.js";
9
+ import IndexEmailAddressWorkflow from "../workflows/email/index/IndexEmailAddressWorkflow.js";
10
10
 
11
11
 
12
12
 
@@ -234,25 +234,14 @@ export default class EmailAddressService implements ICacheContainer<EmailAddress
234
234
  if (name && ea.name !== name) {
235
235
  // schedule for names tag
236
236
  const wc = ServiceProvider.resolve(this, AppWorkflowContext);
237
- await wc.queue(RepairEmailNameTagsWorkflow, { id: ea.emailAddressID, name });
238
- if (name !== username) {
239
- // we should try to save username as well but without stripping numbers
240
- await wc.queue(RepairEmailNameTagsWorkflow, { id: ea.emailAddressID, name:username, includeNumbers: true });
241
- }
237
+ await IndexEmailAddressWorkflow.queue(wc);
242
238
  }
243
239
  if(!hasMailbox && ((name && ea.name !== name) || ea.domainID !== domainID)) {
244
240
  ea.name = name;
245
241
  ea.domainID = domainID;
246
242
 
247
243
  // schedule tag updates...
248
-
249
244
  await db.emailAddresses.statements.update(ea);
250
- const wc = ServiceProvider.resolve(this, AppWorkflowContext);
251
- await wc.queue(RepairEmailNameTagsWorkflow, { id: ea.emailAddressID, name });
252
- if (name !== username) {
253
- // we should try to save username as well but without stripping numbers
254
- await wc.queue(RepairEmailNameTagsWorkflow, { id: ea.emailAddressID, name:username, includeNumbers: true });
255
- }
256
245
  // await db.emailAddresses.saveDirect({
257
246
  // mode: "update",
258
247
  // changes: {
@@ -0,0 +1,65 @@
1
+ import Inject, { RegisterScoped } from "@entity-access/entity-access/dist/di/di.js";
2
+ import SocialMailContext from "../../model/SocialMailContext.js";
3
+ import { Email } from "../../model/entities/Email.js";
4
+ import SearchTokenService from "../search/SearchTokenService.js";
5
+ import { tokenKindType } from "../../model/entities/SearchTokenEmail.js";
6
+ import TextExtractorService from "../extract/TextExtractorService.js";
7
+ import TempFileService from "../../storage/TempFileService.js";
8
+
9
+ @RegisterScoped
10
+ export default class IndexEmailService {
11
+
12
+ @Inject
13
+ db: SocialMailContext;
14
+
15
+ @Inject
16
+ sts: SearchTokenService;
17
+
18
+ @Inject
19
+ te: TextExtractorService;
20
+
21
+ @Inject
22
+ tempFileService: TempFileService;
23
+
24
+ async indexEmail({ emailID, textBody }: Partial<Email>) {
25
+
26
+ // delete all search email tokens if exists
27
+ const { db } = this;
28
+
29
+ await db.searchTokenEmails.asQuery()
30
+ .delete({ emailID }, (p) => (x) => x.emailID === p.emailID);
31
+
32
+ await this.insertText(emailID, textBody, "body");
33
+
34
+ // for all attachments...
35
+ const attachments = await db.appFiles.where({ emailID}, (p) => (x) => x.emailID === p.emailID)
36
+ .toArray();
37
+ for(const att of attachments) {
38
+ const tf = await this.tempFileService.downloadAppFile(att);
39
+ const textFile = await this.te.extract(tf);
40
+ for await(const line of textFile.lines()) {
41
+ if (!line) {
42
+ continue;
43
+ }
44
+ await this.insertText(emailID, line, "attachment");
45
+ }
46
+ }
47
+ }
48
+
49
+
50
+ private async insertText(emailID: number, line: string, tokenKind: tokenKindType) {
51
+ const words = line.match(/\b[\w\.]{2,399}\b/g);
52
+ if (!words) {
53
+ return;
54
+ }
55
+ for (const word of words) {
56
+ const w = word.trim().toLowerCase();
57
+ if (!w) {
58
+ continue;
59
+ }
60
+ const token = await this.sts.getToken(w);
61
+ const ste = { tokenID: token.tokenID, emailID, tokenKind };
62
+ await this.db.searchTokenEmails.statements.selectOrInsert(ste, ste);
63
+ }
64
+ }
65
+ }
@@ -0,0 +1,52 @@
1
+ import Inject, { RegisterScoped } from "@entity-access/entity-access/dist/di/di.js";
2
+ import TempFileService from "../../storage/TempFileService.js";
3
+ import EmailParserService from "../../smtp/services/EmailParserService.js";
4
+ import { LocalFile } from "@entity-access/server-pages/dist/core/LocalFile.js";
5
+ import { parse } from "path";
6
+ import { Extract } from "./Extract.js";
7
+ import { FileType } from "../../../common/FileType.js";
8
+
9
+ @RegisterScoped
10
+ export default class TextExtractorService {
11
+
12
+ @Inject
13
+ tempFileService: TempFileService;
14
+
15
+ @Inject
16
+ parser: EmailParserService;
17
+
18
+ async extract(file: LocalFile) {
19
+
20
+ const { tempFileService, parser } = this;
21
+
22
+ const outputFile = await tempFileService.createTempFile(".txt");
23
+
24
+ const isEmail = file.contentType === "message/rfc822";
25
+ if(isEmail) {
26
+ const parsedEmail = await parser.parse(file, true);
27
+ await outputFile.appendLine(parsedEmail.text);
28
+ for (const iterator of parsedEmail.attachments) {
29
+ const { ext } = parse(iterator.filename || "file.dat");
30
+ const tf = await tempFileService.createTempFile(ext, iterator.filename, iterator.contentType);
31
+ await tf.writeAll(iterator.content);
32
+ const t = await this.extract(tf);
33
+ if (t) {
34
+ await outputFile.appendLine(await t.readAsText());
35
+ }
36
+ }
37
+
38
+ return outputFile;
39
+ }
40
+
41
+ if (/image\//i.test(file.contentType)) {
42
+ await Extract.text(file, outputFile, true);
43
+ return outputFile;
44
+ } if (FileType.canBeCompressed(file)) {
45
+ await Extract.text(file, outputFile);
46
+ return outputFile;
47
+ }
48
+ await outputFile.writeAllText("");
49
+ return outputFile;
50
+ }
51
+
52
+ }
@@ -0,0 +1,33 @@
1
+ import EntityAccessError from "@entity-access/entity-access/dist/common/EntityAccessError.js";
2
+ import { RegisterSingleton, ServiceProvider } from "@entity-access/entity-access/dist/di/di.js";
3
+ import { cacheFor, ICacheContainer, ObjectCache } from "../../../common/cacheFor.js";
4
+ import ReplicaReader from "../replica/ReplicaReader.js";
5
+ import SocialMailContext from "../../model/SocialMailContext.js";
6
+
7
+ @RegisterSingleton
8
+ export default class SearchTokenService implements ICacheContainer<SearchTokenService>{
9
+
10
+ cache: ObjectCache<SearchTokenService>;
11
+
12
+ async getToken(word: string) {
13
+ word = word.trim().toLowerCase();
14
+ if (!word) {
15
+ throw new EntityAccessError("Word cannot be empty");
16
+ }
17
+ return this.getRegisteredToken(word);
18
+ }
19
+
20
+ @cacheFor()
21
+ private async getRegisteredToken(word: string) {
22
+ using scope = ServiceProvider.createScope(this);
23
+ const replica = scope.resolve(ReplicaReader);
24
+ const db = scope.resolve(SocialMailContext);
25
+ let existing = await replica.query((x) => x.searchTokens.statements.select({}, { word }));
26
+ if (existing) {
27
+ return existing;
28
+ }
29
+ existing = await db.searchTokens.statements.selectOrInsert({ word }, { word });
30
+ return existing;
31
+ }
32
+
33
+ }
@@ -55,6 +55,8 @@ import SendLibraryContentWorkflow from "./channel/library/SendLibraryContentWork
55
55
  import CreateChannelWorkflow from "./channel/creation/CreateChannelWorkflow.js";
56
56
  import { AppWorkflowContext } from "./AppWorkflowContext.js";
57
57
  import DeleteFileHistoryWorkflow from "./files/DeleteFileHistoryWorkflow.js";
58
+ import IndexEmailContentWorkflow from "./email/index/IndexEmailContentWorkflow.js";
59
+ import IndexEmailAddressWorkflow from "./email/index/IndexEmailAddressWorkflow.js";
58
60
 
59
61
  @RegisterSingleton
60
62
  export default class SocialWorkflowContext extends AppWorkflowContext {
@@ -121,6 +123,9 @@ export default class SocialWorkflowContext extends AppWorkflowContext {
121
123
  this.register(SendLibraryContentWorkflow);
122
124
  this.register(CreateChannelWorkflow);
123
125
 
126
+ this.register(IndexEmailContentWorkflow);
127
+ this.register(IndexEmailAddressWorkflow);
128
+
124
129
  // start processing...
125
130
  // this.start().catch((error) => console.error(error));
126
131
  }
@@ -13,6 +13,8 @@ import PendingPostReceiveWorkflow from "../pending/PendingPostReceiveWorkflow.js
13
13
  import PendingReferencesWorkflow from "../pending/PendingReferencesWorkflow.js";
14
14
  import MessagingService from "../../services/message-events/MessagingService.js";
15
15
  import FetchCheckoutResultWorkflow from "../store/FetchCheckoutResultWorkflow.js";
16
+ import IndexEmailContentWorkflow from "../email/index/IndexEmailContentWorkflow.js";
17
+ import IndexEmailAddressWorkflow from "../email/index/IndexEmailAddressWorkflow.js";
16
18
 
17
19
  export default class DailyWorkflow extends Workflow<any,any> {
18
20
 
@@ -55,6 +57,8 @@ export default class DailyWorkflow extends Workflow<any,any> {
55
57
 
56
58
  await PendingPostReceiveWorkflow.queue(this.context);
57
59
  await PendingReferencesWorkflow.queue(this.context);
60
+ await IndexEmailContentWorkflow.queue(this.context);
61
+ await IndexEmailAddressWorkflow.queue(this.context);
58
62
 
59
63
  const all = await db.mailboxSyncTargets
60
64
  .where({}, (p) => (x) => x.active === true && x.protocol === "imap" && x.direction === "in")
@@ -0,0 +1,65 @@
1
+ import Inject from "@entity-access/entity-access/dist/di/di.js";
2
+ import Workflow, { UniqueActivity } from "@entity-access/entity-access/dist/workflows/Workflow.js";
3
+ import SocialMailContext from "../../../model/SocialMailContext.js";
4
+ import NameTokenService from "../../../services/name-tokens/NameTokenService.js";
5
+ import WorkflowContext from "@entity-access/entity-access/dist/workflows/WorkflowContext.js";
6
+
7
+ export default class IndexEmailAddressWorkflow extends Workflow {
8
+
9
+ static taskGroup = "batch";
10
+
11
+ static queue(c: WorkflowContext) {
12
+ const d = new Date();
13
+ d.setMinutes(Math.ceil(d.getMinutes()/5));
14
+ d.setSeconds(0);
15
+ d.setMilliseconds(0);
16
+ return c.queue(IndexEmailAddressWorkflow, 0, {
17
+ id: `index-ea-${d.getTime().toString(36)}`
18
+ });
19
+ }
20
+
21
+ async run() {
22
+ let start = 0;
23
+ for(;;) {
24
+ start = await this.index(start);
25
+ if(!BigInt(start)) {
26
+ break;
27
+ }
28
+ }
29
+ }
30
+
31
+ @UniqueActivity
32
+ async index(
33
+ start: number,
34
+ @Inject db?: SocialMailContext,
35
+ @Inject nts?: NameTokenService
36
+ ) {
37
+
38
+ for(let i=0;i<100;i++) {
39
+
40
+ const all = await db.emailAddresses.where({ start }, (p) => (x) =>
41
+ x.nameIndexed === false && x.emailAddressID > p.start)
42
+ .limit(100)
43
+ .toArray();
44
+
45
+ if (!all.length) {
46
+ return 0;
47
+ }
48
+
49
+ for (const { emailAddressID, name, emailAddress } of all) {
50
+ start = emailAddressID;
51
+ if (name) {
52
+ await nts.updateName(name, { emailAddressID });
53
+ }
54
+ const [username ] = emailAddress.split("@", 2);
55
+ await nts.updateName(username, { emailAddressID }, true);
56
+ await db.emailAddresses.statements.update({ nameIndexed: true}, { emailAddressID });
57
+ }
58
+
59
+ db.changeSet.clear();
60
+ }
61
+
62
+ return start;
63
+ }
64
+
65
+ }